{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Description:\n",
    "建立Deep&Cross Model在cretio数据集上进行点击率预测的任务"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T11:58:21.096127Z",
     "start_time": "2020-11-30T11:58:19.217151Z"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm import tqdm\n",
    "import datetime\n",
    "\n",
    "import torch\n",
    "from torch.utils.data import DataLoader, Dataset, TensorDataset\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "\n",
    "from torchkeras import summary, Model\n",
    "\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 数据准备"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T11:58:21.718507Z",
     "start_time": "2020-11-30T11:58:21.704500Z"
    }
   },
   "outputs": [],
   "source": [
    "file_path = './preprocessed_data/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T11:58:22.563276Z",
     "start_time": "2020-11-30T11:58:22.556222Z"
    }
   },
   "outputs": [],
   "source": [
    "def prepared_data(file_path):\n",
    "    \n",
    "    # 读入训练集，验证集和测试集\n",
    "    train = pd.read_csv(file_path + 'train_set.csv')\n",
    "    val = pd.read_csv(file_path + 'val_set.csv')\n",
    "    test = pd.read_csv(file_path + 'test_set.csv')\n",
    "    \n",
    "    trn_x, trn_y = train.drop(columns='Label').values, train['Label'].values\n",
    "    val_x, val_y = val.drop(columns='Label').values, val['Label'].values\n",
    "    test_x = test.values\n",
    "    \n",
    "    fea_col = np.load(file_path + 'fea_col.npy', allow_pickle=True)\n",
    "    \n",
    "    return fea_col, (trn_x, trn_y), (val_x, val_y), test_x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T11:58:24.541288Z",
     "start_time": "2020-11-30T11:58:24.516980Z"
    }
   },
   "outputs": [],
   "source": [
    "fea_cols, (trn_x, trn_y), (val_x, val_y), test_x = prepared_data(file_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T11:58:25.404605Z",
     "start_time": "2020-11-30T11:58:25.395630Z"
    }
   },
   "outputs": [],
   "source": [
    "# 把数据构建成数据管道\n",
    "dl_train_dataset = TensorDataset(torch.tensor(trn_x).float(), torch.tensor(trn_y).float())\n",
    "dl_val_dataset = TensorDataset(torch.tensor(val_x).float(), torch.tensor(val_y).float())\n",
    "\n",
    "dl_train = DataLoader(dl_train_dataset, shuffle=True, batch_size=32)\n",
    "dl_val = DataLoader(dl_val_dataset, shuffle=True, batch_size=32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T06:50:30.922459Z",
     "start_time": "2020-11-30T06:50:30.912487Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([32, 39]) tensor([0., 0., 1., 0., 1., 0., 0., 0., 0., 0., 1., 1., 1., 0., 0., 0., 1., 1.,\n",
      "        0., 0., 0., 0., 1., 0., 0., 0., 0., 1., 0., 0., 1., 0.])\n"
     ]
    }
   ],
   "source": [
    "# 看一下数据\n",
    "for b in iter(dl_train):\n",
    "    print(b[0].shape, b[1])\n",
    "    break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 构建模型\n",
    "这里依然是使用继承nn.Module基类构建模型， 并辅助应用模型容器进行封装， Deep&Cross的模型结构如下：\n",
    "\n",
    "![](img/deep&Cross.png)\n",
    "\n",
    "这个模型W&D的基础上， 保持了Deep部分不变， 把Wide部分换成了Cross network的形式， 所以这里的关键部分是实现CrossNetwork， 也就是那个交叉公式。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T11:59:29.292773Z",
     "start_time": "2020-11-30T11:59:29.283798Z"
    }
   },
   "outputs": [],
   "source": [
    "class CrossNetwork(nn.Module):\n",
    "    \"\"\"\n",
    "    Cross Network\n",
    "    \"\"\"\n",
    "    def __init__(self, layer_num, input_dim):\n",
    "        super(CrossNetwork, self).__init__()\n",
    "        self.layer_num = layer_num\n",
    "        \n",
    "        # 定义网络层的参数\n",
    "        self.cross_weights = nn.ParameterList([\n",
    "            nn.Parameter(torch.rand(input_dim, 1))\n",
    "            for i in range(self.layer_num)\n",
    "        ])\n",
    "        self.cross_bias = nn.ParameterList([\n",
    "            nn.Parameter(torch.rand(input_dim, 1))\n",
    "            for i in range(self.layer_num)\n",
    "        ])\n",
    "    \n",
    "    def forward(self, x):\n",
    "        # x是(None, dim)的形状， 先扩展一个维度到(None, dim, 1)\n",
    "        x_0 = torch.unsqueeze(x, dim=2)\n",
    "        x = x_0.clone()\n",
    "        xT = x_0.clone().permute((0, 2, 1))     # （None, 1, dim)\n",
    "        for i in range(self.layer_num):\n",
    "            x = torch.matmul(torch.bmm(x_0, xT), self.cross_weights[i]) + self.cross_bias[i] + x   # (None, dim, 1)\n",
    "            xT = x.clone().permute((0, 2, 1))   # (None, 1, dim)\n",
    "        \n",
    "        x = torch.squeeze(x)  # (None, dim)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T11:59:30.662111Z",
     "start_time": "2020-11-30T11:59:30.644161Z"
    }
   },
   "outputs": [],
   "source": [
    "class Dnn(nn.Module):\n",
    "    \"\"\"\n",
    "    Dnn part\n",
    "    \"\"\"\n",
    "    def __init__(self, hidden_units, dropout=0.):\n",
    "        \"\"\"\n",
    "        hidden_units: 列表， 每个元素表示每一层的神经单元个数， 比如[256, 128, 64], 两层网络， 第一层神经单元128， 第二层64， 第一个维度是输入维度\n",
    "        dropout: 失活率\n",
    "        \"\"\"\n",
    "        super(Dnn, self).__init__()\n",
    "        \n",
    "        self.dnn_network = nn.ModuleList([nn.Linear(layer[0], layer[1]) for layer in list(zip(hidden_units[:-1], hidden_units[1:]))])\n",
    "        self.dropout = nn.Dropout(p=dropout)\n",
    "    \n",
    "    def forward(self, x):\n",
    "        \n",
    "        for linear in self.dnn_network:\n",
    "            x = linear(x)\n",
    "            x = F.relu(x)\n",
    "        \n",
    "        x = self.dropout(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T11:59:32.229920Z",
     "start_time": "2020-11-30T11:59:32.212966Z"
    }
   },
   "outputs": [],
   "source": [
    "class DCN(nn.Module):\n",
    "    def __init__(self, feature_columns, hidden_units, layer_num, dnn_dropout=0.):\n",
    "        super(DCN, self).__init__()\n",
    "        self.dense_feature_cols, self.sparse_feature_cols = feature_columns\n",
    "        \n",
    "        # embedding \n",
    "        self.embed_layers = nn.ModuleDict({\n",
    "            'embed_' + str(i): nn.Embedding(num_embeddings=feat['feat_num'], embedding_dim=feat['embed_dim'])\n",
    "            for i, feat in enumerate(self.sparse_feature_cols)\n",
    "        })\n",
    "        \n",
    "        hidden_units.insert(0, len(self.dense_feature_cols) + len(self.sparse_feature_cols)*self.sparse_feature_cols[0]['embed_dim'])\n",
    "        self.dnn_network = Dnn(hidden_units)\n",
    "        self.cross_network = CrossNetwork(layer_num, hidden_units[0])         # layer_num是交叉网络的层数， hidden_units[0]表示输入的整体维度大小\n",
    "        self.final_linear = nn.Linear(hidden_units[-1]+hidden_units[0], 1)\n",
    "    \n",
    "    def forward(self, x):\n",
    "        dense_input, sparse_inputs = x[:, :len(self.dense_feature_cols)], x[:, len(self.dense_feature_cols):]\n",
    "        sparse_inputs = sparse_inputs.long()\n",
    "        sparse_embeds = [self.embed_layers['embed_'+str(i)](sparse_inputs[:, i]) for i in range(sparse_inputs.shape[1])]\n",
    "        sparse_embeds = torch.cat(sparse_embeds, axis=-1)\n",
    "        \n",
    "        x = torch.cat([sparse_embeds, dense_input], axis=-1)\n",
    "        \n",
    "        # cross Network\n",
    "        cross_out = self.cross_network(x)\n",
    "        \n",
    "        # Deep Network\n",
    "        deep_out = self.dnn_network(x)\n",
    "\n",
    "        #  Concatenate\n",
    "        total_x = torch.cat([cross_out, deep_out], axis=-1)\n",
    "        \n",
    "        # out\n",
    "        outputs = F.sigmoid(self.final_linear(total_x))\n",
    "        \n",
    "        return outputs  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T11:59:56.689516Z",
     "start_time": "2020-11-30T11:59:56.664585Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------------------------\n",
      "        Layer (type)               Output Shape         Param #\n",
      "================================================================\n",
      "         Embedding-1                    [-1, 8]             632\n",
      "         Embedding-2                    [-1, 8]           2,016\n",
      "         Embedding-3                    [-1, 8]          10,344\n",
      "         Embedding-4                    [-1, 8]           8,344\n",
      "         Embedding-5                    [-1, 8]             240\n",
      "         Embedding-6                    [-1, 8]              56\n",
      "         Embedding-7                    [-1, 8]           9,312\n",
      "         Embedding-8                    [-1, 8]             312\n",
      "         Embedding-9                    [-1, 8]              16\n",
      "        Embedding-10                    [-1, 8]           7,264\n",
      "        Embedding-11                    [-1, 8]           7,408\n",
      "        Embedding-12                    [-1, 8]           9,912\n",
      "        Embedding-13                    [-1, 8]           6,592\n",
      "        Embedding-14                    [-1, 8]             160\n",
      "        Embedding-15                    [-1, 8]           6,552\n",
      "        Embedding-16                    [-1, 8]           9,272\n",
      "        Embedding-17                    [-1, 8]              72\n",
      "        Embedding-18                    [-1, 8]           4,272\n",
      "        Embedding-19                    [-1, 8]           1,608\n",
      "        Embedding-20                    [-1, 8]              32\n",
      "        Embedding-21                    [-1, 8]           9,632\n",
      "        Embedding-22                    [-1, 8]              56\n",
      "        Embedding-23                    [-1, 8]              96\n",
      "        Embedding-24                    [-1, 8]           5,832\n",
      "        Embedding-25                    [-1, 8]             264\n",
      "        Embedding-26                    [-1, 8]           4,432\n",
      "           Linear-27                  [-1, 128]          28,416\n",
      "           Linear-28                   [-1, 64]           8,256\n",
      "           Linear-29                   [-1, 32]           2,080\n",
      "          Dropout-30                   [-1, 32]               0\n",
      "           Linear-31                    [-1, 1]             254\n",
      "================================================================\n",
      "Total params: 143,734\n",
      "Trainable params: 143,734\n",
      "Non-trainable params: 0\n",
      "----------------------------------------------------------------\n",
      "Input size (MB): 0.000149\n",
      "Forward/backward pass size (MB): 0.003548\n",
      "Params size (MB): 0.548302\n",
      "Estimated Total Size (MB): 0.551998\n",
      "----------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "# 建立模型\n",
    "hidden_units = [128, 64, 32]\n",
    "dnn_dropout = 0.\n",
    "\n",
    "model = DCN(fea_cols, hidden_units, len(hidden_units), dnn_dropout)\n",
    "summary(model, input_shape=(trn_x.shape[1],))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T11:59:59.119020Z",
     "start_time": "2020-11-30T11:59:59.093090Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[1.0000e+00],\n",
      "        [0.0000e+00],\n",
      "        [9.9797e-01],\n",
      "        [7.9662e-15],\n",
      "        [1.0000e+00],\n",
      "        [0.0000e+00],\n",
      "        [1.0000e+00],\n",
      "        [1.0000e+00],\n",
      "        [0.0000e+00],\n",
      "        [0.0000e+00],\n",
      "        [0.0000e+00],\n",
      "        [1.0000e+00],\n",
      "        [1.0000e+00],\n",
      "        [9.7600e-01],\n",
      "        [1.0000e+00],\n",
      "        [1.0000e+00],\n",
      "        [1.0000e+00],\n",
      "        [1.0000e+00],\n",
      "        [4.2269e-36],\n",
      "        [0.0000e+00],\n",
      "        [0.0000e+00],\n",
      "        [1.2843e-35],\n",
      "        [7.1598e-01],\n",
      "        [0.0000e+00],\n",
      "        [3.1315e-01],\n",
      "        [1.0000e+00],\n",
      "        [3.4485e-26],\n",
      "        [0.0000e+00],\n",
      "        [1.0000e+00],\n",
      "        [1.0000e+00],\n",
      "        [1.0000e+00],\n",
      "        [0.0000e+00]], grad_fn=<SigmoidBackward>)\n"
     ]
    }
   ],
   "source": [
    "# 测试一下模型\n",
    "for fea, label in iter(dl_train):\n",
    "    out = model(fea)\n",
    "    print(out)\n",
    "    break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 模型的训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T12:00:01.821835Z",
     "start_time": "2020-11-30T12:00:01.803842Z"
    }
   },
   "outputs": [],
   "source": [
    "# 模型的相关设置\n",
    "def auc(y_pred, y_true):\n",
    "    pred = y_pred.data\n",
    "    y = y_true.data\n",
    "    return roc_auc_score(y, pred)\n",
    "\n",
    "loss_func = nn.BCELoss()\n",
    "optimizer = torch.optim.Adam(params=model.parameters(), lr=0.001)\n",
    "metric_func = auc\n",
    "metric_name = 'auc'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T12:00:21.689668Z",
     "start_time": "2020-11-30T12:00:02.777241Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start_training.........\n",
      "================================================================2020-11-30 20:00:02\n",
      "[step=10] loss: 11.396, auc: 0.530\n",
      "[step=20] loss: 11.415, auc: 0.551\n",
      "[step=30] loss: 11.576, auc: 0.545\n",
      "[step=40] loss: 11.417, auc: 0.543\n",
      "\n",
      "EPOCH=1, loss=11.417, auc = 0.543, val_loss=10.878, val_auc = 0.518\n",
      "\n",
      "================================================================================2020-11-30 20:00:05\n",
      "[step=10] loss: 11.152, auc: 0.521\n",
      "[step=20] loss: 10.971, auc: 0.529\n",
      "[step=30] loss: 10.046, auc: 0.556\n",
      "[step=40] loss: 9.987, auc: 0.557\n",
      "\n",
      "EPOCH=2, loss=9.987, auc = 0.557, val_loss=9.479, val_auc = 0.526\n",
      "\n",
      "================================================================================2020-11-30 20:00:07\n",
      "[step=10] loss: 8.484, auc: 0.576\n",
      "[step=20] loss: 9.267, auc: 0.549\n",
      "[step=30] loss: 9.431, auc: 0.565\n",
      "[step=40] loss: 9.471, auc: 0.567\n",
      "\n",
      "EPOCH=3, loss=9.471, auc = 0.567, val_loss=9.510, val_auc = 0.528\n",
      "\n",
      "================================================================================2020-11-30 20:00:09\n",
      "[step=10] loss: 8.579, auc: 0.593\n",
      "[step=20] loss: 8.857, auc: 0.580\n",
      "[step=30] loss: 8.663, auc: 0.577\n",
      "[step=40] loss: 8.755, auc: 0.552\n",
      "\n",
      "EPOCH=4, loss=8.755, auc = 0.552, val_loss=8.946, val_auc = 0.565\n",
      "\n",
      "================================================================================2020-11-30 20:00:11\n",
      "[step=10] loss: 9.005, auc: 0.585\n",
      "[step=20] loss: 8.557, auc: 0.580\n",
      "[step=30] loss: 8.686, auc: 0.569\n",
      "[step=40] loss: 8.309, auc: 0.571\n",
      "\n",
      "EPOCH=5, loss=8.309, auc = 0.571, val_loss=8.722, val_auc = 0.584\n",
      "\n",
      "================================================================================2020-11-30 20:00:12\n",
      "[step=10] loss: 9.126, auc: 0.572\n",
      "[step=20] loss: 8.343, auc: 0.589\n",
      "[step=30] loss: 8.443, auc: 0.592\n",
      "[step=40] loss: 8.303, auc: 0.584\n",
      "\n",
      "EPOCH=6, loss=8.303, auc = 0.584, val_loss=8.582, val_auc = 0.495\n",
      "\n",
      "================================================================================2020-11-30 20:00:14\n",
      "[step=10] loss: 6.925, auc: 0.649\n",
      "[step=20] loss: 7.648, auc: 0.615\n",
      "[step=30] loss: 7.544, auc: 0.610\n",
      "[step=40] loss: 7.577, auc: 0.610\n",
      "\n",
      "EPOCH=7, loss=7.577, auc = 0.610, val_loss=8.870, val_auc = 0.525\n",
      "\n",
      "================================================================================2020-11-30 20:00:16\n",
      "[step=10] loss: 7.948, auc: 0.626\n",
      "[step=20] loss: 7.820, auc: 0.605\n",
      "[step=30] loss: 7.674, auc: 0.615\n",
      "[step=40] loss: 7.533, auc: 0.620\n",
      "\n",
      "EPOCH=8, loss=7.533, auc = 0.620, val_loss=8.208, val_auc = 0.517\n",
      "\n",
      "================================================================================2020-11-30 20:00:18\n",
      "[step=10] loss: 6.261, auc: 0.677\n",
      "[step=20] loss: 6.678, auc: 0.642\n",
      "[step=30] loss: 6.985, auc: 0.641\n",
      "[step=40] loss: 7.201, auc: 0.627\n",
      "\n",
      "EPOCH=9, loss=7.201, auc = 0.627, val_loss=7.891, val_auc = 0.532\n",
      "\n",
      "================================================================================2020-11-30 20:00:19\n",
      "[step=10] loss: 7.999, auc: 0.582\n",
      "[step=20] loss: 7.260, auc: 0.595\n",
      "[step=30] loss: 6.806, auc: 0.614\n",
      "[step=40] loss: 6.632, auc: 0.615\n",
      "\n",
      "EPOCH=10, loss=6.632, auc = 0.615, val_loss=7.994, val_auc = 0.566\n",
      "\n",
      "================================================================================2020-11-30 20:00:21\n",
      "Finished Training\n"
     ]
    }
   ],
   "source": [
    "# 脚本训练风格\n",
    "epochs = 10\n",
    "log_step_freq = 10\n",
    "\n",
    "dfhistory = pd.DataFrame(columns=['epoch', 'loss', metric_name, 'val_loss', 'val_'+metric_name])\n",
    "\n",
    "print('start_training.........')\n",
    "nowtime = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')\n",
    "print('========'*8 + '%s' %nowtime)\n",
    "\n",
    "for epoch in range(1, epochs+1):\n",
    "    \n",
    "    # 训练阶段\n",
    "    model.train()\n",
    "    loss_sum = 0.0\n",
    "    metric_sum = 0.0\n",
    "    step = 1\n",
    "    \n",
    "    for step, (features, labels) in enumerate(dl_train, 1):\n",
    "        # 梯度清零\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        # 正向传播\n",
    "        predictions = model(features);\n",
    "        loss = loss_func(predictions, labels)\n",
    "        try:\n",
    "            metric = metric_func(predictions, labels)\n",
    "        except ValueError:\n",
    "            pass\n",
    "        \n",
    "        # 反向传播\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        # 打印batch级别日志\n",
    "        loss_sum += loss.item()\n",
    "        metric_sum += metric.item()\n",
    "        if step % log_step_freq == 0:\n",
    "            print((\"[step=%d] loss: %.3f, \" + metric_name + \": %.3f\") % (step, loss_sum/step, metric_sum/step));\n",
    "    \n",
    "    # 验证阶段\n",
    "    model.eval()\n",
    "    val_loss_sum = 0.0\n",
    "    val_metric_sum = 0.0\n",
    "    val_step = 1\n",
    "    \n",
    "    for val_step, (features, labels) in enumerate(dl_val, 1):\n",
    "        with torch.no_grad():\n",
    "            predictions = model(features)\n",
    "            val_loss = loss_func(predictions, labels)\n",
    "            try:\n",
    "                val_metric = metric_func(predictions, labels)\n",
    "            except ValueError:\n",
    "                pass\n",
    "        \n",
    "        val_loss_sum += val_loss.item()\n",
    "        val_metric_sum += val_metric.item()\n",
    "    \n",
    "    # 记录日志\n",
    "    info = (epoch, loss_sum/step, metric_sum/step, val_loss_sum/val_step, val_metric_sum/val_step)\n",
    "    dfhistory.loc[epoch-1] = info\n",
    "    \n",
    "    # 打印日志\n",
    "    print((\"\\nEPOCH=%d, loss=%.3f, \" + metric_name + \" = %.3f, val_loss=%.3f, \" + \"val_\" + metric_name + \" = %.3f\") %info)\n",
    "    nowtime = datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")\n",
    "    print('\\n' + '=========='* 8 + '%s' %nowtime)\n",
    "    \n",
    "print('Finished Training')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T12:00:30.260750Z",
     "start_time": "2020-11-30T12:00:30.239806Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>epoch</th>\n",
       "      <th>loss</th>\n",
       "      <th>auc</th>\n",
       "      <th>val_loss</th>\n",
       "      <th>val_auc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>11.416629</td>\n",
       "      <td>0.543086</td>\n",
       "      <td>10.877676</td>\n",
       "      <td>0.518139</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2.0</td>\n",
       "      <td>9.987390</td>\n",
       "      <td>0.556751</td>\n",
       "      <td>9.478765</td>\n",
       "      <td>0.526419</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>3.0</td>\n",
       "      <td>9.471452</td>\n",
       "      <td>0.567479</td>\n",
       "      <td>9.509526</td>\n",
       "      <td>0.527518</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>8.755144</td>\n",
       "      <td>0.552272</td>\n",
       "      <td>8.945654</td>\n",
       "      <td>0.564516</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>5.0</td>\n",
       "      <td>8.308987</td>\n",
       "      <td>0.570606</td>\n",
       "      <td>8.721694</td>\n",
       "      <td>0.583606</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>6.0</td>\n",
       "      <td>8.302937</td>\n",
       "      <td>0.583695</td>\n",
       "      <td>8.582025</td>\n",
       "      <td>0.495121</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.577258</td>\n",
       "      <td>0.609883</td>\n",
       "      <td>8.870394</td>\n",
       "      <td>0.524995</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.532686</td>\n",
       "      <td>0.620332</td>\n",
       "      <td>8.207756</td>\n",
       "      <td>0.517269</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>9.0</td>\n",
       "      <td>7.200949</td>\n",
       "      <td>0.627224</td>\n",
       "      <td>7.890959</td>\n",
       "      <td>0.531727</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>10.0</td>\n",
       "      <td>6.631770</td>\n",
       "      <td>0.615392</td>\n",
       "      <td>7.993854</td>\n",
       "      <td>0.565940</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   epoch       loss       auc   val_loss   val_auc\n",
       "0    1.0  11.416629  0.543086  10.877676  0.518139\n",
       "1    2.0   9.987390  0.556751   9.478765  0.526419\n",
       "2    3.0   9.471452  0.567479   9.509526  0.527518\n",
       "3    4.0   8.755144  0.552272   8.945654  0.564516\n",
       "4    5.0   8.308987  0.570606   8.721694  0.583606\n",
       "5    6.0   8.302937  0.583695   8.582025  0.495121\n",
       "6    7.0   7.577258  0.609883   8.870394  0.524995\n",
       "7    8.0   7.532686  0.620332   8.207756  0.517269\n",
       "8    9.0   7.200949  0.627224   7.890959  0.531727\n",
       "9   10.0   6.631770  0.615392   7.993854  0.565940"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfhistory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T12:00:32.671305Z",
     "start_time": "2020-11-30T12:00:32.396041Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEWCAYAAABhffzLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXhU5fXA8e8JAUJARAGRRRLcAFmVKCiKCy4UcamKImjB3aIFrRuCW1Wq1daFalVURCUiij9cKy4gYhXEQEFAiyiGRUE22WQRyPn9cSYSwkwySWZyZzLn8zzzZObOnXtPJnDuve/73vOKquKccy51pAUdgHPOucrlid8551KMJ37nnEsxnvidcy7FeOJ3zrkU44nfOedSjCd+V2EiUk1ENolI81iuGyQROVhEYj7WWUROFpH8Iq8XiMhx0axbjn09IyJDy/v5ErZ7r4iMjvV2XeVJDzoAV/lEZFORl5nANmBn6PVVqppblu2p6k6gTqzXTQWq2jIW2xGRy4GLVPWEItu+PBbbdlWPJ/4UpKq/Jd7QGeXlqvphpPVFJF1Vd1RGbM65+POmHreH0KX8OBEZKyIbgYtE5GgRmS4i60RkuYiMEJHqofXTRURFJDv0ekzo/XdFZKOITBORFmVdN/T+70TkGxFZLyL/FJFPRWRAhLijifEqEflWRH4WkRFFPltNRB4WkTUi8h3Qo4Tv5zYRebnYssdF5KHQ88tF5OvQ7/Nd6Gw80raWicgJoeeZIvJiKLb5QKcw+10U2u58ETkztLwd8BhwXKgZbXWR7/auIp+/OvS7rxGR10WkcTTfTWlE5OxQPOtEZLKItCzy3lAR+VFENojI/4r8rl1EZFZo+U8i8mC0+3MxoKr+SOEHkA+cXGzZvcCvwBnYyUEt4EigM3aVeCDwDXBtaP10QIHs0OsxwGogB6gOjAPGlGPd/YCNwFmh9/4MbAcGRPhdoonxDWBvIBtYW/i7A9cC84FmQH1gqv33CLufA4FNQO0i214J5IRenxFaR4CTgC1A+9B7JwP5Rba1DDgh9PzvwBRgHyAL+KrYuucDjUN/k76hGBqF3rscmFIszjHAXaHnp4Zi7AhkAP8CJkfz3YT5/e8FRoeetw7FcVLobzQ09L1XB9oAi4H9Q+u2AA4MPf8CuDD0fC+gc9D/F1Lp4Wf8LpL/qOpbqlqgqltU9QtV/VxVd6jqImAkcHwJnx+vqnmquh3IxRJOWdftBcxW1TdC7z2MHSTCijLG+1R1varmY0m2cF/nAw+r6jJVXQPcX8J+FgHzsAMSwCnAOlXNC73/lqouUjMZmASE7cAt5nzgXlX9WVUXY2fxRff7iqouD/1NXsIO2jlRbBegH/CMqs5W1a3AEOB4EWlWZJ1I301J+gBvqurk0N/ofqAudgDegR1k2oSaC78PfXdgB/BDRKS+qm5U1c+j/D1cDHjid5EsLfpCRFqJyDsiskJENgB3Aw1K+PyKIs83U3KHbqR1mxSNQ1UVO0MOK8oYo9oXdqZakpeAC0PP+2IHrMI4eonI5yKyVkTWYWfbJX1XhRqXFIOIDBCROaEmlXVAqyi3C/b7/bY9Vd0A/Aw0LbJOWf5mkbZbgP2NmqrqAuAG7O+wMtR0uH9o1UuAw4AFIjJDRHpG+Xu4GPDE7yIpPpTxKews92BVrQvcgTVlxNNyrOkFABERdk9UxVUkxuXAAUVelzbcdBxwcuiM+SzsQICI1ALGA/dhzTD1gPejjGNFpBhE5EDgCeCPQP3Qdv9XZLulDT39EWs+KtzeXliT0g9RxFWW7aZhf7MfAFR1jKp2xZp5qmHfC6q6QFX7YM15/wBeE5GMCsbiouSJ30VrL2A98IuItAauqoR9vg0cISJniEg6MBhoGKcYXwGuE5GmIlIfuKWklVX1J+A/wHPAAlVdGHqrJlADWAXsFJFeQPcyxDBUROqJ3edwbZH36mDJfRV2DLwcO+Mv9BPQrLAzO4yxwGUi0l5EamIJ+BNVjXgFVYaYzxSRE0L7vgnrl/lcRFqLyImh/W0JPXZiv8DFItIgdIWwPvS7FVQwFhclT/wuWjcA/bH/1E9hZ7xxFUquFwAPAWuAg4D/YvcdxDrGJ7C2+LlYx+P4KD7zEtZZ+1KRmNcB1wMTsA7S87ADWDTuxK488oF3gReKbPdLYAQwI7ROK6Bou/gHwELgJxEp2mRT+PmJWJPLhNDnm2Pt/hWiqvOx7/wJ7KDUAzgz1N5fE3gA65dZgV1h3Bb6aE/ga7FRY38HLlDVXysaj4uOWLOpc4lPRKphTQvnqeonQcfjXLLyM36X0ESkh4jsHWouuB0bKTIj4LCcS2qe+F2iOxZYhDUX9ADOVtVITT3OuSh4U49zzqUYP+N3zrkUkxRF2ho0aKDZ2dlBh+Gcc0ll5syZq1V1jyHQSZH4s7OzycvLCzoM55xLKiIS9g50b+pxzrkU44nfOedSjCd+55xLMUnRxu+cq3q2b9/OsmXL2Lp1a9ChJL2MjAyaNWtG9eqRSjXtzhO/cy4Qy5YtY6+99iI7OxsrvOrKQ1VZs2YNy5Yto0WLFqV/gCrc1JObC9nZkJZmP3PLNH24cy7etm7dSv369T3pV5CIUL9+/TJdOVXJM/7cXLjySti82V4vXmyvAfpVuB6hcy5WPOnHRlm/xyp5xj9s2K6kX2jzZlvunHOprkom/iVLyrbcOedSSZVM/M0jTJoXablzLvHFut9u3bp1/Otf/yrz53r27Mm6devK/LkBAwYwfnw08/vEX5VM/MOHQ2bm7stq1LDlzrnkU9hvt3gxqO7qt6tI8o+U+Hfu3Fni5/79739Tr1698u84AVTJxN+vH4wcCVlZIALVq0PdunDuuUFH5pyL5IQT9nwU5uVbbw3fbzd4sD1fvXrPz5ZmyJAhfPfdd3Ts2JEjjzySE088kb59+9KuXTsAzj77bDp16kSbNm0YOXLkb5/Lzs5m9erV5Ofn07p1a6644gratGnDqaeeypYtW6L6XSdNmsThhx9Ou3btuPTSS9m2bdtvMR122GG0b9+eG2+8EYBXX32Vtm3b0qFDB7p16xbV9ktTJRM/WPLPz4eCAnj3XfuHkSBXWc65MloWYUr4NWvKv83777+fgw46iNmzZ/Pggw8yY8YMhg8fzldffQXAqFGjmDlzJnl5eYwYMYI1YXa2cOFCrrnmGubPn0+9evV47bXXSt3v1q1bGTBgAOPGjWPu3Lns2LGDJ554grVr1zJhwgTmz5/Pl19+yW232fTEd999N++99x5z5szhzTffLP8vXESVHM5ZXPfuMG0adO4cdCTOuUimTIn8XvPm1rxTXFaW/WzQoOTPR+Ooo47a7QaoESNGMGHCBACWLl3KwoULqV+//m6fadGiBR07dgSgU6dO5Ofnl7qfBQsW0KJFCw499FAA+vfvz+OPP861115LRkYGl19+Oaeffjq9evUCoGvXrgwYMIDzzz+fc845p2K/ZEiVPeMvrksXa/YpR5+Mcy5g4frtMjNj229Xu3bt355PmTKFDz/8kGnTpjFnzhwOP/zwsDdI1axZ87fn1apVY8eOHaXuJ9Ksh+np6cyYMYNzzz2X119/nR49egDw5JNPcu+997J06VI6duwY9sqjrFIm8QO8/z40awazZgUdiXOuLIr322Vl2euK3JC51157sXHjxrDvrV+/nn322YfMzEz+97//MX369PLvqJhWrVqRn5/Pt99+C8CLL77I8ccfz6ZNm1i/fj09e/bkkUceYfbs2QB89913dO7cmbvvvpsGDRqwdOnSCseQEk09hTp3trOE666Djz+2f0DOueTQr19s77yvX78+Xbt2pW3bttSqVYtGjRr99l6PHj148sknad++PS1btqRLly4x229GRgbPPfccvXv3ZseOHRx55JFcffXVrF27lrPOOoutW7eiqjz88MMA3HTTTSxcuBBVpXv37nTo0KHCMSTFZOs5OTkaqxm4Ro6Eq66CV16B3r1jsknnXDl8/fXXtG7dOugwqoxw36eIzFTVnOLrplRTD8Bll0H79nDTTRDlyCvnnKtSUi7xV6sGjzwCS5fC5MlBR+Ocq2quueYaOnbsuNvjueeeCzqs3aRUG3+hE0+Eb7+FKEtXO+dc1B5//PGgQyhVyp3xFypM+t9/H2wczjlX2eKW+EVklIisFJF5RZb1FpH5IlIgInt0OFS2l1+Ggw+GL74IOhLnnKs88TzjHw30KLZsHnAOMDWO+41az552x99111nhJ+ecSwVxS/yqOhVYW2zZ16q6IF77LKu6de3Ov88+g3Hjgo7GOecqR8K28YvIlSKSJyJ5q1atitt+LrkEOnaEm2/24Z3OJbSAJ9KuU6dOxPfy8/Np27ZtJUZTMQmb+FV1pKrmqGpOw4YN47afatXg0Ufh55+9lINzCSseBflTWEoO5yyuWzcb15/kcys4l7yuuw5CtWnCmj4dQjXrf7N5s92R+fTT4T/TsaPdtBPBLbfcQlZWFgMHDgTgrrvuQkSYOnUqP//8M9u3b+fee+/lrLPOKtOvsnXrVv74xz+Sl5dHeno6Dz30ECeeeCLz58/nkksu4ddff6WgoIDXXnuNJk2acP7557Ns2TJ27tzJ7bffzgUXXFCm/ZVHwp7xV1gZLwvr1bMTiRjWYnLOxUrxpF/a8ij06dOHcUU691555RUuueQSJkyYwKxZs/joo4+44YYbIlbTjKRwHP/cuXMZO3Ys/fv3Z+vWrTz55JMMHjyY2bNnk5eXR7NmzZg4cSJNmjRhzpw5zJs377eKnHGnqnF5AGOB5cB2YBlwGfD70PNtwE/Ae9Fsq1OnTlomY8aoZmaqWi63R2amLS/Bv/5lq06fXrbdOefK7quvvop+5ays3f8/Fz6ysioUQ6tWrfSHH37Q2bNn6zHHHKO//vqrXnPNNdquXTvt0KGDZmRk6PLly1VVtXbt2hG38/3332ubNm1UVfXss8/WSZMm/fbescceq3PmzNHc3Fw97LDD9P7779dvvvlGVVUXLFig2dnZevPNN+vUqVMr9LuE+z6BPA2TU+M5qudCVW2sqtVVtZmqPquqE0LPa6pqI1U9LS47HzYs/Dxtw4aV+LGLLoL99/fhnc4lnDgV5D/vvPMYP34848aNo0+fPuTm5rJq1SpmzpzJ7NmzadSoUdg6/CXRCMmjb9++vPnmm9SqVYvTTjuNyZMnc+ihhzJz5kzatWvHrbfeyt13312h3ydaVbOpZ8mSsi0P2Wsv+Otfrbln7Ng4xOWcK594FOTHmntefvllxo8fz3nnncf69evZb7/9qF69Oh999BGLw037VYpu3bqRG2pa/uabb1iyZAktW7Zk0aJFHHjggQwaNIgzzzyTL7/8kh9//JHMzEwuuugibrzxRmZV0giTqtm5G2metubNS/1o//7w2GNwyy1w9tl7nmQ45wIS64L8QJs2bdi4cSNNmzalcePG9OvXjzPOOIOcnBw6duxIq1atyrzNgQMHcvXVV9OuXTvS09MZPXo0NWvWZNy4cYwZM4bq1auz//77c8cdd/DFF19w0003kZaWRvXq1XniiSdi+vtFUjXr8RcO/Sra3JOWBs8/b+05pfjkE1vt7behXbtyBOycK5XX44+tstTjr5pn/IVnBcOGWfPOvvvCmjWwc2dUHz/uOKveWb16HGN0zrmAVM02frDkn58PBQWwcqXNu3jLLbBhQ1Qfr14dtm6FN96Ib5jOueQxd+7cPWrtd+7cOeiwyqxqnvEXl5YG//wnHHUU3HMPPPhgVB979FEYMsRq+Rx9dJxjdC4FqSqSRJNft2vX7rdJ0BNJWZvsq+4Zf3FHHgmXXmrZfEF0deKuuQYaN7bhnQUFcY7PuRSTkZHBmjVrypy03O5UlTVr1pCRkRH1Z6pm524kP/0Ehx4KxxwD//63DQsrxfPPw4AB8MILcPHFFQ/BOWe2b9/OsmXLyjxO3u0pIyODZs2aUb1Yx2Skzt3USvwADz0EN9wAb70FvXqVunpBAXTpAj/8AN98A7VrxyYM55yLt0iJP3Waegpdey20amXtN1HU+UhLszpPBxxgfcTOOZfsUi/x16hh7fzffQcPPxzVR445BqZN88nZnXNVQ+olfoBTT4WzzoJ774Uff4zqIyJ2xv/MM3GOzTnn4iw1Ez9YW/+OHTa2P0pPPglXXGHDO51zLlmlbuI/8EDr5B0zBj79NKqP3HADNG0Kgwf78E7nXPJK3cQPcOutlskHDYqqnEPt2nD//ZCXBy++WAnxOedcHKR24q9Tx+7inTULRo2K6iN9+9oNwLfeCps2xTk+55yLg9RO/AB9+sCxx8LQobBuXamrp6XZoKBu3WDLlkqIzznnYswTv4jV8Vm7Fu66K6qPdOkCL78MDRvGNzTnnIsHT/wAHTta/f7HHoP586P+2Pz51ubvnHPJxBN/oXvusbkXBw+OesLdV1+1tv5PPolzbM45F0Oe+As1aGDJf9IkmDAhqo/cfDM0a+bVO51zycUTf1FXXw1t29qA/Sh6bjMz4W9/s0FBzz9fCfE551wMeOIvKj0dRoywmbv+/veoPnLhhTZJy9ChsHFjfMNzzrlY8MRf3IknQu/ecN99Nl9vKUSseueAAVGV93fOucB54g/nwQetg/emm6Ja/aij7DhRp06c43LOuRjwxB9OVpZNtvvKKzBlStQfe/996/B1zrlE5ok/kptvtgPA4MFWxTMKn39uFwsffxzn2JxzrgI88UdSqxb84x/w5ZcwcmRUH7nxRpup67rroqr55pxzgfDEX5JzzoGTToLbboM1a0pdvVYteOABmD0bRo+Of3jOOVcenvhLImIV2TZsgNtvj+ojF1wAXbva8M4NG+Icn3POlYMn/tK0bQsDB8JTT8GcOaWuXnisuOceq9/vnHOJRjTKujRBysnJ0by8vOAC+PlnOOQQaNPGRvn4gH3nXBIQkZmqmlN8uZ/xR2OffeCvf4WpU22IZ5RGjbI5ep1zLpHELfGLyCgRWSki84os21dEPhCRhaGf+8Rr/zF32WVw+OE2dOeXX6L6yPLl8Mwz0KiRTeCSnQ25ufEN0znnShPPM/7RQI9iy4YAk1T1EGBS6HVyqFbNJmxZtizqIvxNmlir0MqVdiPw4sVW9t+Tv3MuSHFL/Ko6FVhbbPFZQGEdy+eBs+O1/7jo2tUm3X3wQVi0qNTV//KXPUv7b94Mw4bFKT7nnItCZbfxN1LV5QChn/tFWlFErhSRPBHJW7VqVaUFWKoHHrAqnjfeWOqqkWq8RVH7zTnn4iZhO3dVdaSq5qhqTsNEmty2aVM7ZZ8wAT74oMRVmzcPv3y//aKe5Ms552KushP/TyLSGCD0c2Ul7z82rr8eDjrI6vhs3x5xteHDbbKWomrWhJ9+surPCxbEOU7nnAujshP/m0D/0PP+wBuVvP/YyMiAhx+Gr7+Gxx+PuFq/flbmJyvLOnmzsmyUz9NP271g7dvD3XfDtm2VGLtzLuXF7QYuERkLnAA0AH4C7gReB14BmgNLgN6qWrwDeA+B38AVjir07AmffQYLF1r7TRmsWGEXDi+/DF262Gb8vjDnXCxFuoHL79ytiP/9D9q1s+m3nn66XJuYOBHWrYM+fWzC9o0bYe+9Yxumcy41+Z278dCqlbXzP/sslPPA1KOHJX2wO31btrSrgCQ4HjvnkpQn/oq64w5r5hk0yE7ZKyAnx+r5X3ghnH66zfnunHOx5om/ourWtTt5p02r8C25HTvC9Ok2efvUqVYTbtSoGMXpnHMhnvhj4Q9/sBnXb77ZGukroFo1az366ivo3h0aN45RjM45F+KJPxbS0mDECBuqM3x4TDbZvDm8+Sb87nf2+q67bErHCh5XnHPOE3/MdO5so3seesiGd8bYunV2bGnTBt56K+abd86lEE/8sXTffXZz1/XXx3zTjzwCn35qQz3PPBPOOw9+/DHmu3HOpQBP/LG0//5w553wzjv2iLGjj4aZM21OmPfeg9WrY74L51wK8Bu4Yu3XX60WQ0EBzJsHNWrEZTfr1kG9evb8vvvgjDNsemDnnCvkN3BVlho1rF1m4UKbdT1OCpP+qlXWrXD44VY0dMuWuO3SOVdFeOKPhx497BT87rtt/sU4atjQasX162dNQO3bw6RJcd2lcy7JeeKPl4cesmafIfGfXbJBAxg9Gj780F6fey6sX19spdxcm/TXJ/91LuV54o+Xgw+GG26AF16wTt9KSLjdu8OXX8L779von4ICePtt0Nxcm+x38WKf/Nc5R3rQAVRpBx9stZZ/+sleFyZcsLaZaKjC1q02We+WLbv/DLOs1ubNHLVlC7y+mUVfbuaHd7awNS2XWgWbd99u4eS/0cbhnKsyPPHH0913h59t/eqrrV2mlCT+2/PySE/noMxMGtfNJGPDL2FX0cVL8CkAnEs9nvjjKdKs6ps2WQ9sZqY9atWC2rWtp7ZWrV3Lir5f/HlJ79eqBdWrI0BtYFl6Ns12Lt4jjLVp9akf1y/AOZeIPPHHU/Pm1rxTXFZWpdZcvmXncEZyJbXZ1dyzkzTqF6yGvn1t+sh99qm0eJxzwfLO3XgKN9t6ZmbMCrlF69OsflzBSPLJogAhnyz6M5q/1bkHXn3VZhF7//1Kjck5FxxP/PEUbrb1kSMrvUN1+HB4I7MfLcinGgW0IJ+X5GJu33YbUx/83IYAnXYaXHMN/BK+P8A5V3V44o+3fv2sWaegwH4GMIom3PHnscegQwc48YYjGHnVTPjzn+GJJ+wW4OnTKz1G51zl8cSfIooffwYOhClToGdPuGpwBsMy/gGTJ8O2bdC1K9x+u92A5pyrcjzxp7DatWHCBDsIHHQQcMIJMHcu9O8P994LXbrA/PlBh+mcizFP/CkuPd0G9Vx6qb2enFeXtX8fBa+/DsuWQadO8I9/VHgieedc4vDE736zfr3V+TnmGPi+/VlWVrpHD7jxRjjppEodguqcix9P/O43e+9t8/yuXGmtPHlL9rO2oNGjYdYsK/05atSedyM755KKJ363m+OOg88+s9sNjj8e3n5HrM1/7lxr9rnsMjj77F31h5xzSccTv9tDq1Y2ovOww4rU9s/KshcPP2zzPrZrZ1cDzrmk44nfhdWoEXz8Mfz97/b6hx+ggDS47jpr9jngADjnHBgwIEzxf+dcIosq8YvIYBGpK+ZZEZklIqfGOzgXrMxMqFYNfv4ZOne2sj5bt2KXAtOn21j/MWPs7H/y5KDDdc5FKdoz/ktVdQNwKtAQuAS4P25RuYRSrx4MHgzjxsGpp8LatUD16lZ2+tNPrRpo9+5w/fU+6a9zSSDaxF9Ytr0n8JyqzimyzFVxInDTTfDyy/D556Hhnt+H3uzcGf77X/jTn2yS+SOOgLy8QON1zpUs2sQ/U0TexxL/eyKyF+B39KSYCy6w+WNWroShQ4u8kZkJI0ZYhc+NG+Hoo+1qYPv2wGJ1zkUWbeK/DBgCHKmqm4HqWHNPuYT6DOaJyHwRua6823GV77jjYNo0ePJJe71zZ5E3TznFhn326QN33mk1fxYsCCRO51xk0Sb+o4EFqrpORC4CbgPKNZRDRNoCVwBHAR2AXiJySHm25YLRsqXd7LVlC5x4ohX1/M0++8CLL1qd/0WLoGNH+Oc/veSDcwkk2sT/BLBZRDoANwOLgRfKuc/WwHRV3ayqO4CPgd+Xc1suQAUFdgAYOBBuuaVYbj/vPDv7P+kkGDTIeoWXLoXcXMjOhrQ0+5mbG1D0zqWuaBP/DlVV4CzgUVV9FNirnPucB3QTkfoikon1GxxQfCURuVJE8kQkb9WqVeXclYunwuqef/wjPPBAkeGehRo3hrfftskApk+3S4VLL7XpKFXt55VXevJ3rpKJRlF3RUQ+BiYClwLHAauA2ararlw7FbkMuAbYBHwFbFHV6yOtn5OTo3k+UiRhqdqNXjffDBdfDC+EuxZctMjG/2/btud7lTwHsXOpQkRmqmpO8eXRnvFfAGzDxvOvAJoCD5Y3GFV9VlWPUNVuwFpgYXm35YJXONzzlVfgttsirHTggZEndlm8OPyk9K5yePNbyokq8YeSfS6wt4j0Araqannb+BGR/UI/mwPnAGPLuy2XOHr3hkMPtSuAm24KM5y/efPIH87OtiJBgwZZ89CmTfEM1RXKzbXmNm9+SynRlmw4H5gB9AbOBz4XkfMqsN/XROQr4C3gGlX9uQLbcglm1SoYPz5U3fPtIm8MH25j/ovKzLQOgkcesWnAnn0WzjgD9t3XZgT761/tCLLbuFEXM8OGwebNuy/bvNmWuyor2jb+OcApqroy9Loh8KGqdohzfIC38SejFSugVy+7qfexx6wDGLAzyWHDYMkSuwIYPnz3Cei3bbO60O+/b49Zs2x5/fpw8sk2OuiUU6xInKuYpUsjX4WJ+BDcKiBSG3+0iX9u0Y5cEUkD5pS3c7esPPEnp02b4MIL7az/zjvhrrvKsZGVK60c9Hvv2YFg+XJb3rq1HQROPdUuLWrXjmXoVdfatXY59tJLMHVq5El1mjf3fpcqoKKduxNF5D0RGSAiA4B3gH/HMkBX9dSpY8M9r73WqjhAOfoR99vPjh6jR1tt6LlzbQ7g5s3hqafg9NOtWeikk+D+++0Kwc9Ud7d5sxVaOvNM2H9/uOoquyT7y1/suyze/AZ2IF29uvJjdZUiqjN+ABE5F+iKFWebqqqVNguHn/FXDbm5Noy/6OCezEwb5l+0tSdqW7fCf/6zq1lozhxb3qCBNQcVNgs1bRqT+JPK9u1WWOmll+zo+8sv9j1ceKHdcNGxozXnwJ7Nbz162IF2//3h9ddtXZeUIp3xo6oJ/+jUqZO65Ne0qaq1Lez+yMqK0Q6WL1d98UXViy9WbdRo1w7atFG9/nrVd99V/eUX1TFjbKci9nPMmBgFELCdO1X/8x/VgQNVGzSw371ePdUrrlD96CPVHTui39aMGfYHq1VL9eWX4xayiy8gT8Pk1BLP+EVkIxBuBbFjhtaN1ZGpJH7GXzWkpYVvUo5LP6KqNQsVXg1MnWodx9Wq2XtFd1ihy44EMG+endm/9JK1y9eqZc06ffvCaadBzZrl2+6KFXDuudbZfuutcM899v25pFGhzt2geeKvGu/5BLoAABShSURBVLKzw/cXHnCAtTLE1ZYt8MknVkNo48Y938/MtIJDbdpA27Y2tDQ9Pc5BVcDixTB2rCX7uXMtIZ9yiiX7s8+GvcpbUaWYbdtsroWnn4aePW1/e+8dm227uPPE7wJXeK9Q0WHjNWva0P2+fe39886DjIw4BhHpsgPs0qPwvZo17Yaytm3tUXhAyMqybQRh9WqrevrSS9a3ATYrTt++dvfcfvvFZ7+qVod70CC7A/uNN+y7cQnPE79LCJGG8U+bZjmseXNrUejXL06tCpEuO7Ky4Kuv4Ouvrelk/nz7OW+ejXcvVLu21RwqfkBo0mRXZ2ksbdpkifall6zJascO23+/ftZR26JF7PcZydSpdmTets3+kL16Vd6+Xbl4565LeJMnq+bkWJ9khw6q770Xh52MGaOambl773JmZskdvOvWqX72merIkaqDBql2775753FhJ2rXrqpXXaX6z3/aL7NyZemxhOtk3rZN9a23VC+8cFesBxygesstqnPmqBYUxOrbKLvFi1WPOMJiHj482FhcqShP526i8DP+1FFQYMXehg61E8vvvotD009pdw9Ha/Xq3a8MCp//XKQCyX777boqKLxCaNMG3nknfLtX164we7bdaLXvvnD++daU07VrcE1MxW3eDJdfbn0MvXvDc8/5DXQJypt6XFLZtg0WLrRc+euvMGSINTFnZwcdWSlU7e7i4geE+fN3LzxXrVr4+kMiu8ban3IK1KhRebGXhYZqcQ8ZYn+k11+v3GYnFxVP/C5pTZtmN+YWFNhdwMOG2clwUikosCuMwgPCkCHh10u2GjnvvWdzLFerZpdqJ50UdESuiIqWbHAuMEcfbWf/F120q4jnAw/YzalJo7BGxemn27DRrKzw65VUujoRnXYazJhhTVqnngojRkQeNeUShid+lxSaNbNhn3PmWHP3uHFJfi9RpBLVw4cHE09FHHKITa15+ukweLDV5dhtDk6XaDzxu6TStq1V+/zoIzuJXrPGinNOnJhkJ5r9+tndwllZ1ryTlZXcdw/XrWs1ge64w+r8nHAC/Phj0FG5CDzxu6RUN1QsZPFiWLYMfvc76wstLN+fFPr1s7mGCwrsZ7Im/UJpaVbx87XXrB+jUyfroHEJxxO/S2pHHGH3XD36qDUDdepk+TOp2v+rmnPOsYSfmWln/qNGBR2RK8YTv0t6NWrYUM9vv7Xx/wDVq9tPb2oOSLt28MUX0K0bXHaZ1fvxo3HC8MTvqoy997a+0TFj7PWCBVYA7m9/sxptrpLtuy+8+y78+c82/+app9qEzC5wnvhdlVNYMic9Hbp0sSHzhx5qfY4+Z3slS0+3Wb5eeMGaf4480u5MdoHyxO+qrIMOgrfegilToHFjuOQSOxC8+GIZp3+MkzJPQ5nMLr7YymLv2GHV+MaNCzqilOaJ31V5xx8Pn39uN5YefDBcfbWNBlK1n1dcYS0RK1bs6hPYvt1K7qxfb+X7f/nFmosKrxgK53Ip7xDSwhLVReO48soqnvyPPBLy8qxHvk8f65DxS7BAeOJ3KUHE6olNm7Z7XTSwhP6nP9lVwbvv2rJJk6yJul49Gzpap44NUpk82d4fP95uIEtLs0d6unUyT59u77/4os2FsvfesM8+UL8+NGxoI5DAOqOLx7F5s5WjqNL239++xCuvhPvus5nC1q0LOqqUk8BTDDkXeyXN9PXEE7vmFW/VyspDFBTs/jj4YHu/TRsbsl78/SZN7P2WLS23FX+/Xj17v2gBz2jjqzJq1ICnnoLDD7cjbufOPrlLJfMibS6llDQPS35+6sURuKKTu1xxhV1KVbRctvuNF2lzjsQpkVNSHMWbgKq0bt2s3X+ffWz0T0p1egTHE79LKYlSIidSHD16WAvI8OFJVnuoIpo3D1+KOiU6PYLhTT3OJZBff7UbXceMgauustFG6anQE5eWFvlIN22a9QPEY07jKs6bepxLAjVq2L1OQ4ZY/+c556RI00+keQhEbEKGdu3g4YdtuktXYZ74nUswIjbS8fHHbWre668POqJKEKnTY+RIe9SpY6UfmjSxcbkTJ/o9ABWQCheRziWlgQNt9E+nTkFHUgkKO1mGDQs/queKK6zU86hRdkk0frwVYhowwG7J9vl+y8Tb+J1LAtu328RWgwbZDbApbds2ePNNm5Lt/fetb6B7d+sc+f3vISMj6AgTRkK18YvI9SIyX0TmichYEfG/lHMl+Okn+PRTK2//zjtBRxOwmjV3Nffk59uddN9+C337WlPQn/5kkzO4iCo98YtIU2AQkKOqbYFqQJ/KjsO5ZNKsmQ1uad0azjrLTnYd1iR0xx2waBF88IFN/j5ypN2CnZNjt2Mna0mIOFbxC6pzNx2oJSLpQCbgk3M6V4pGjazS6CmnwOWXw0MPBR1RAklLg5NPhrFjba7fRx+19rGBA60I08UX25eXBE3bQNyr+AXSxi8ig4HhwBbgfVXd4/YZEbkSuBKgefPmnRaHu7/duRS0fbuN9LnsMrvZy0WgCjNn2uXRSy/Bhg1Wq/vSS6F/f2jaNNj4CgqsJGx+Pnz/vf0sfEyZYiWsiytjTY9IbfyVnvhFZB/gNeACYB3wKjBeVcdE+ox37joX2XPPWZN3nTpBR5LANm+2SeCffRY+/tiuEH73Ozt69uq1a67OWCoosM6Zogm9aIJfvNju2CuqUSNr1vn88/DbFAl/l3MEiZT4ewM9VPWy0Os/AF1UdWCkz3jidy68+fOhfXs783/nHcsbrhTffmvDQkePhuXLYb/94A9/sINAq1bWnBJpWGlRqrBy5Z5n60Uf27bt/pmGDW3oaXb2no+srF33MsSoil8iJf7OwCjgSKypZzSQp6r/jPQZT/zORfb223DBBZb0J060aSZdFHbssC/s2WdtqradO+GQQ/Y8E8/IsE6VZs32TOyFM/cUatBgVyIvnuCzsqB27ehiK2zjL3rbduENbWUoLJUwiT8UzF+wpp4dwH+By1V1W6T1PfE7V7IZM6zFoqDADgRdugQdUZJZscJuDBs2LHzbeqH69fc8Uy9M8FlZsW1vi/bKowQJlfjLyhO/c6X79lvo2RPuvRfOPz/oaJJUpGJxIjYstG7dyo+pAiIlfi/Z4FwVcfDBMHeu3d8EsHChtVy4MmjePHzbevPmSZf0S+JF2pyrQgqT/syZNj3k0KHJM3Q9ISTKTD1x5onfuSqoQwerXXbffTZkvfioQRdBoszUE2fe1ONcFZSeDk8+aQUsb7/dRi2+9lqVaq2In379qlyiL87P+J2rokTgttvsBq8pU+D554OOyCUKP+N3roobMMBu8Grf3l7v3AnVqgUakguYn/E7lwI6dLArgO++s07fTz4JOiIXJE/8zqWQtND/+FNOsTZ/l5o88TuXQlq0sAldOnWywm6PPhp0RC4InvidSzH168OHH8LZZ8N118FVV8Vtvg+XoLxz17kUVKsWvPqqzU/y4ouwZYstL5zvA6r8iMaU5mf8zqWoatXgs892Jf1CmzdbMco+feDdd23Zzz/DI49YNePx422O8+nTYc0ae7+gwEYLVUQcZxp0xfgZv3MpbMmS8Mu3boX//tfmKgFYutRm/SruuedsuOiMGXD00Vacsm7dXY+//c0miP/qK3jqqd3f22sv6N7dykk/8wwMGuRXHpXFE79zKSxSTbKsLFiwYNfrtm3trH/Dht0fbdva+40bw5137vl+Ye2gJUus6vGGDbtPIDVliiX+W28Nf+UxbJgn/njwsszOpbAYzfcRNVXb14YNsH69lZSoXbvkashlmGnQFROpLLO38TuXwiq7JpmIJfrGjW2Ww8IJqZo3D79+4fI//9n6GNaujU9cqcYTv3Mprl8/m0WwoMB+BtG0UlI15O3b4YsvrI+haVPrU5g2zctNV4Qnfudc4Eq68qhe3UpMzJ5tpaZfew2OOQb+9a+go05e3sbvnEsqGzfC2LE2x3CTJjZP+ltvwdVXwxFHBB1dYvE2fudclbDXXtYh3aSJvf72WxgzxspQHHWU3Wvwyy/BxpjoPPE755La9dfDjz/CiBE2Yuiyy+DEE4OOKrF54nfOJb169eBPf7LJ5j/5BP7yF1u+ZQv07GlXBFu3BhtjIvHE75yrMkTg2GN33XH8/fewcKHVJGraFG68Eb75JtgYE4EnfudclXXYYXYH8ocfwkknWRnqli1h/vygIwuWJ37nXJWWlmY1gV591WoOPfGEHRAAhg61shD5+bvWT4VicV6rxzmXMvbf34Z9FsrPh3Hj4L77oEcPaN0annxyVwmLqloszsfxO+dS2tKlVh30mWdsdFA4WVm7XxUkCx/H75xzYRxwgI0CWrzYOofDiVS+Oll54nfOOSA9PXKxuGbNKjeWePPE75xzIeGKxYEViqtKrc2e+J1zLiRcsbhbb7WrgaOPhg8+CDrC2PBRPc45V0S/fnuO4Ln5ZptG8rjj7PXOnTZncbLyM37nnCtFvXo25DMjw2YP69jR5hBOgkGRYVV64heRliIyu8hjg4hcV9lxOOdceWzZYjOIXX21lYZesSLoiMqu0hO/qi5Q1Y6q2hHoBGwGJlR2HM45Vx6NGsHEiVYNdPJkm3D+//4v6KjKJuimnu7Ad6q6OOA4nHMuamlpVg101iwr6zBiRHI1+wTdudsHGBvuDRG5ErgSoHmkwbXOOReg1q1t/t/1620U0PLlVg20W7egIytZYGf8IlIDOBN4Ndz7qjpSVXNUNadhw4aVG5xzzkWpenVo0MCe33knnHAC3HILbNsWaFglCrKp53fALFX9KcAYnHMuZh56CK64Ah54wKaBnDs36IjCCzLxX0iEZh7nnEtGderYMM+33rLRPjk58M47QUe1p0ASv4hkAqcASdYX7pxzpevVC+bNgz/8we74hcTq/A0k8avqZlWtr6rrg9i/c87FW8OG8PTTsO++Vuvn5JPhhRcS4wAQ9HBO55yr8tatg19/hf79oXdvWL062Hg88TvnXJw1bAhTpsD998Obb0K7dnYTWFA88TvnXCWoVs2Gec6YAfXrw403wo4dwcQS9A1czjmXUjp2tNr+K1ZYueeNG2HBAhsBVFn8jN855ypZRoaVegC45x7o0sWmf9y+vXL274nfOecCNHQo9O0Ld90Fxx4L33wT/3164nfOuQDVq2fDPF95Bb791pqC3ngDcnPtqiAtzX7m5sZun574nXMuAfTubSUeevaE77+HK6+ExYtt3P/ixfY6VslfNBHuJihFTk6O5lWlmY6dc64E2dmW7IvLyoL8/Oi3IyIzVXWPbmM/43fOuQSzZEnZlpeVJ37nnEswkaYgidXUJJ74nXMuwQwfDpmZuy/LzLTlseCJ3znnEky/fjBypLXpi9jPkSNteSz4nbvOOZeA+vWLXaIvzs/4nXMuxXjid865FOOJ3znnUownfuecSzGe+J1zLsUkRckGEVkFhLmBOak0AAKecC2h+Pexi38Xu/PvY3cV+T6yVLVh8YVJkfirAhHJC1czI1X597GLfxe78+9jd/H4PrypxznnUownfuecSzGe+CvPyKADSDD+fezi38Xu/PvYXcy/D2/jd865FONn/M45l2I88TvnXIrxxB9nInKAiHwkIl+LyHwRGRx0TEETkWoi8l8ReTvoWIImIvVEZLyI/C/0b+TooGMKiohcH/o/Mk9ExopIRtAxVSYRGSUiK0VkXpFl+4rIByKyMPRzn1jsyxN//O0AblDV1kAX4BoROSzgmII2GPg66CASxKPARFVtBXQgRb8XEWkKDAJyVLUtUA3oE2xUlW400KPYsiHAJFU9BJgUel1hnvjjTFWXq+qs0PON2H/spsFGFRwRaQacDjwTdCxBE5G6QDfgWQBV/VVV1wUbVaDSgVoikg5kAj8GHE+lUtWpwNpii88Cng89fx44Oxb78sRfiUQkGzgc+DzYSAL1CHAzUBB0IAngQGAV8Fyo6esZEakddFBBUNUfgL8DS4DlwHpVfT/YqBJCI1VdDnYSCewXi4164q8kIlIHeA24TlU3BB1PEESkF7BSVWcGHUuCSAeOAJ5Q1cOBX4jRpXyyCbVdnwW0AJoAtUXkomCjqro88VcCEamOJf1cVf2/oOMJUFfgTBHJB14GThKRMcGGFKhlwDJVLbwCHI8dCFLRycD3qrpKVbcD/wccE3BMieAnEWkMEPq5MhYb9cQfZyIiWBvu16r6UNDxBElVb1XVZqqajXXcTVbVlD2rU9UVwFIRaRla1B34KsCQgrQE6CIimaH/M91J0Y7uYt4E+oee9wfeiMVGfbL1+OsKXAzMFZHZoWVDVfXfAcbkEsefgFwRqQEsAi4JOJ5AqOrnIjIemIWNhPsvKVa6QUTGAicADURkGXAncD/wiohchh0ce8dkX16ywTnnUos39TjnXIrxxO+ccynGE79zzqUYT/zOOZdiPPE751yK8cTvUpqI7BSR2UUeMbtzVkSyi1ZadC5R+Dh+l+q2qGrHoINwrjL5Gb9zYYhIvoj8TURmhB4Hh5ZnicgkEfky9LN5aHkjEZkgInNCj8JyA9VE5OlQnfn3RaRWaP1BIvJVaDsvB/RruhTlid+lulrFmnouKPLeBlU9CngMqypK6PkLqtoeyAVGhJaPAD5W1Q5YvZ35oeWHAI+rahtgHXBuaPkQ4PDQdq6O1y/nXDh+565LaSKySVXrhFmeD5ykqotCRfZWqGp9EVkNNFbV7aHly1W1gYisApqp6rYi28gGPghNooGI3AJUV9V7RWQisAl4HXhdVTfF+Vd17jd+xu9cZBrheaR1wtlW5PlOdvWrnQ48DnQCZoYmH3GuUnjidy6yC4r8nBZ6/hm7pgTsB/wn9HwS8Ef4bU7hupE2KiJpwAGq+hE2KU09YI+rDufixc8yXKqrVaRqKtj8t4VDOmuKyOfYCdKFoWWDgFEichM2e1ZhNc3BwMhQFcWd2EFgeYR9VgPGiMjegAAPp/iUi66SeRu/c2GE2vhzVHV10LE4F2ve1OOccynGz/idcy7F+Bm/c86lGE/8zjmXYjzxO+dcivHE75xzKcYTv3POpZj/B0J+zULWffa0AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dd3hUddbA8e8hdJRFsSEtuKIIgigR7BJcFXdd1BUVRVd8BRsKllWxLGBhdRVXxbYCFhQUBAvqYkPBCkhQUAGVDhHU0Dsk5Lx/nAkZwiRMkpncmcn5PM88mXvnlpPJZM69vyqqinPOOVdUlaADcM45l5g8QTjnnIvIE4RzzrmIPEE455yLyBOEc865iDxBOOeci8gThKswIpImIhtFpEkstw2SiBwqIjFvKy4ifxKRxWHLP4nIydFsW4ZzDReRO8u6v0tdVYMOwCUuEdkYtlgb2AbsCC1fraqjSnM8Vd0B7BXrbSsDVT08FscRkZ7AparaMezYPWNxbJd6PEG4Yqnqzi/o0BVqT1WdWNz2IlJVVfMqIjbnXPx5EZMrMxG5X0TGiMirIrIBuFREjheRqSKyVkRWiMgQEakW2r6qiKiIpIeWR4Zef09ENojIFBFpVtptQ6+fJSI/i8g6EXlCRL4UkR7FxB1NjFeLyHwRWSMiQ8L2TRORR0VklYgsADqX8P7cLSKji6x7SkT+E3reU0Tmhn6fBaGr++KOlS0iHUPPa4vIy6HYZgPtIpx3Yei4s0WkS2h9a+BJ4ORQ8d3KsPd2YNj+14R+91Ui8paINIjmvSnl+7xb0ZyIfBH+Nwud58fQ7/GDiBxV3LlcnKiqP/yxxwewGPhTkXX3A9uBv2IXG7WAY4EO2N3pIcDPwPWh7asCCqSHlkcCK4EMoBowBhhZhm0PADYA54ReuxnIBXoU87tEE+N44A9AOrC64HcHrgdmA42A+sBn9m8U8TyHABuBOmHH/h3ICC3/NbSNAJ2ALUCb0Gt/AhaHHSsb6Bh6PhiYDOwDNAXmFNn2QqBB6G9ySSiGA0Ov9QQmF4lzJDAw9PyMUIxtgZrA08An0bw3pXyfDy36vgFfFPzNgIuBZVjyE+AwoHHQ/weV7eF3EK68vlDVd1Q1X1W3qOp0VZ2mqnmquhAYCpxawv7jVDVLVXOBUdgXU2m3PRuYqarjQ689iiWTiKKM8QFVXaeqi7Ev44JzXQg8qqrZqroKeLCE8ywEfsASF8DpwFpVzQq9/o6qLlTzCfAxELEiuogLgftVdY2qLsHuCsLP+5qqrgj9TV7BkntGFMcF6A4MV9WZqroV6AecKiKNwrYp7r3ZRRk+C+F6Ag+q6ozQ+/Ozqi6Lcl8XI54gXHnt8k8rIi1E5H8i8quIrAfuBfYrYf9fw55vpuSK6eK2PTg8DlVV7Io7oihjjOpcwJIS4gV4BbsaBrua31mxLyJni8g0EVktImuxq/eS3qsCDUqKQUR6iMisUNHOWqBFlMcF+/12Hk9V1wNrgIZh20T1NyvDZyFcY2BBlNu6OPEE4cqraBPPZ7Gr5kNVtS7QHysiiKcVWJEPACIi7PqFVlR5YlyBfXkV2FMz3DHAn0JX4OdgCQMRqQWMAx7Ain/qAR9GGcevxcUgIocAzwDXAvVDx/0x7Lh7apK7HCu2Kjje3lhR1i9RxFVUSe/zptDxa4dtf1DY82XAH8twThdDniBcrO0NrAM2icgRwNUVcM53gWNE5K8iUhXoC+wfpxhfA24UkYYiUh+4vaSNVfU3rGz9BeAnVZ0XeqkGUB3IAXaIyNnAaaWI4U4RqSfWT+T6sNf2wpJADpYre2J3EAV+AxoVVBZH8CpwpYi0EZEaWAL7XFWLvSMrQUnv86+hx6Whiv+rCEtMwHDgNhE5WkxzEQlPiq4CeIJwsXYLcDlWafwsdgUdV6Ev4YuA/wCrsCvPb7F+G7GO8RmsruB7YDp2F7Anr2CVzq+ExbwWuAl4E6vo7YolumgMwO5kFgPvAS+FHfc7YAjwdWibFsC0sH0/AuYBv4lIeFFRwf7vY0VBb4b2b4LVS5RFse9zqBiwF3AnVl90aHicqvoq8O/QPuuBN7A7GVeBxP5OzqUOEUnDikq6qurnQcfjXLLyOwiXEkSks4j8IVQs8k8gD7uKds6VkScIlypOAhZixRWdgXNVtbgiJudcFLyIyTnnXER+B+Gccy6ilBmsb7/99tP09PSgw3DOuaQyY8aMlaoasVl4yiSI9PR0srKygg7DOeeSiogUOxqAFzE555yLyBOEc865iDxBOOeciyhl6iAiyc3NJTs7m61btwYdSlKrWbMmjRo1olq14obvcc6lopROENnZ2ey9996kp6djA3y60lJVVq1aRXZ2Ns2aNdvzDs65lJHSRUxbt26lfv36nhzKQUSoX7++34U5F2bUKEhPhypV7OeoUXvaIzml9B0E4MkhBvw9dK7QqFFw1VWwebMtL1liywDdyzrubYJK6TsI55yLtbvuKkwOBTZvtvWpJuXvIJxzLhays+Gdd+yOIZKlSys2norgdxBh4lGuuHbtWp5++ulS7/fnP/+ZtWvXlj8A51y5fP01tGsHjRvDdddB1WIuq9PSYORI2LGjYuOLJ08QIQXlikuWgGphuWJ5k0RxCWLHHj5FEyZMoF69euU7uXOuVLZtgw8/hOuvh7fesnUHHAA1asCDD8KcOfDii1C79q771agBBx8Ml10GRx0FU6ZUeOhxUakSRMeOuz8KvrvvuCNyuWLfvvZ85crd941Gv379WLBgAW3btuXYY48lMzOTSy65hNatWwNw7rnn0q5dO1q1asXQoUN37peens7KlStZvHgxRxxxBL169aJVq1acccYZbNmypdjzDRs2jGOPPZajjjqK888/n82hX6pHjx6MG1c4O+Zee+218/lDDz1E69atOeqoo+jXr190v5hzKULVrvwvvBD23x/OPBNeeAF+/tleT0+Hr76C22+HI46wiuihQ6FpUxCxn889B4sWwWuvQX4+FPx7bdlix09aqpoSj3bt2mlRc+bM2WX51FN3fzz1lL0momp/yt0fqqo5ObvvG41FixZpq1atVFV10qRJWrt2bV24cOHO11etWqWqqps3b9ZWrVrpypUrVVW1adOmmpOTo4sWLdK0tDT99ttvVVX1ggsu0JdffrnY8xXsr6p611136ZAhQ1RV9fLLL9exY8fufK1OnTqqqjphwgQ9/vjjddOmTbvEU1TR99K5ZDZvnur48YXLRx6p2qCB6lVXqb77rurmzWU/dn5+4fNu3VQzM1WnTCn78eINyNJivlcrVSX15MnFv9akSeTKp6ZN7ed++5W8f7Tat2+/S4ezIUOG8OabbwKwbNky5s2bR/369XfZp1mzZrRt2xaAdu3asXjx4mKP/8MPP3D33Xezdu1aNm7cyJlnnlliPBMnTuSKK66gduieed999y3Lr+VcQtuxA6ZNg7fftsfcuXaVv2oVVK9uxUoHHmj1j+UV3ir8hBPg/vvh+OOhSxe47z5o06b856golaqIqSSDBu1erli7tq2PpTp16ux8PnnyZCZOnMiUKVOYNWsWRx99dMQOaTVq1Nj5PC0tjby8vGKP36NHD5588km+//57BgwYsPN4VatWJT8/H7C7xu3bt+987v0cXCratAkK/lXuvx9OPBEeecTqCh5/HL77zpIDQIMGsUkORd1wAyxYYN8jn34KbdvCSy/F/jzx4gkiJFK54tCh5e/4svfee7Nhw4aIr61bt4599tmH2rVr8+OPPzJ16tTynQzYsGEDDRo0IDc3l1FhNezp6enMmDEDgPHjx5ObmwvAGWecwfPPP7+zrmL16tXljsG5oCxfbv+3Z59td/2TJtn6iy6CV1+FnByYOBH69IGKGjlmr73gzjutjuLOO62OA6zCOzu7YmIoq7gmCBHpLCI/ich8EYlY+ykiF4rIHBGZLSKvhNa1FZEpoXXfichF8YyzQPfusHixVTItXhybXpH169fnxBNP5Mgjj+TWW2/d5bXOnTuTl5dHmzZt+Oc//8lxxx1X7vPdd999dOjQgdNPP50WLVrsXN+rVy8+/fRT2rdvz7Rp03beyXTu3JkuXbqQkZFB27ZtGTx4cLljcC5eimuKvmIFtG8PDRvC1VfD7Nn2s1Eje71FC+jWDYJsGLjPPnYnc+CBtty7Nxx6KNxyiyWuhFRc5UR5H0AasAA4BKgOzAJaFtmmOfAtsE9o+YDQz8OA5qHnBwMrgHolnS+aSmpXdv5euqCNHKlau/auDUiqVbP1eXmqZ56pev/9qt9/v2tFcaJatEj1iitUq1RR3Wsv1f79Vdeurfg4KKGSOp53EO2B+aq6UFW3A6OBc4ps0wt4SlXXAKjq76GfP6vqvNDz5cDvQMQ5U51zlUOkpui5uTbERVoavP++PT/yyF0rihNVejo8/7zd7Zx1Ftx7rzWvTSTxbMXUEFgWtpwNdCiyzWEAIvIldscxUFXfD99ARNpjdyALip5ARK4CrgJo0qRJzAJPBr179+bLL7/cZV3fvn254oorAorIufhatizy+mQf4qJFC+s/8c031s8C4I034NdfoWfPwor0IMQzQUTK4UW7jFTFipk6Ao2Az0XkSFVdCyAiDYCXgctVNX+3g6kOBYYCZGRkJHN3lFJ76qmngg7BubjKzrbWRpdfbncFDRpYXUNRqXJteMwxhc/ffNM67w0eDPfcA5dcYndJFS2eRUzZQOOw5UbA8gjbjFfVXFVdBPyEJQxEpC7wP+BuVS1/8x7nXFKYMweuuAIOOQQefRS++MLWP/xwxTRFTwQvvQTvvWeV6n//u/WdmDix4uOIZ4KYDjQXkWYiUh3oBrxdZJu3gEwAEdkPK3JaGNr+TeAlVR0bxxidcwlCFS6+GFq1gjFj4JprYP58+wnxa4qeiESgc2fIyoKxY62j37p19lpeXsUN3xG3BKGqecD1wAfAXOA1VZ0tIveKSJfQZh8Aq0RkDjAJuFVVVwEXAqcAPURkZujRNl6xOueCkZ9vV8aq9qXYogUMHGj1CkOGWEVuuHg0RU9kVapA167www/wt7/ZugcegE6dbHyoeM9sF9ehNlR1AjChyLr+Yc8VuDn0CN9mJDAynrE554KzbZuVsT/8MPz0kw1jc+qpMGBA0JElpvAhxg8+2IYKOfFEq5coGBg6HjPbeU/qcAkw0Wz4KKvOpZotW+Chh6wXc8+eVofw6qv2Zeeic+WVNnxHvXq7zz0R65ntKtVgfSWqTBPNOlfBcnOhWjUrRvrPf6B1axgxAv70p+Tos5Bo6tQprJMoKpbNfitPgrjxRpg5s/jXp061+95wmzdbuh42LPI+bdvCY4+VeNrbb7+dpk2bct111wEwcOBARITPPvuMNWvWkJuby/3338855xTtQ7i7jRs3cs455+y23+LFizn77LP54YcfABg8eDAbN25k4MCBzJ8/n2uuuYacnBzS0tIYO3Ysf/zjH/d4Ludi4ccfranmF19YOXrNmvZzv/2Cjiz5FTcCdSyb/XoRU4GiyWFP66PUrVs3xowZs3P5tdde44orruDNN9/km2++YdKkSdxyyy0FQ4+UqGbNmqXer3v37vTu3ZtZs2bx1Vdf0aBBg3L9Ps5F46uv4NxzrePXqFFw2mmFN+eeHGKjIkagrjx3EHu40ic9vfgJIcoxEcTRRx/N77//zvLly8nJyWGfffahQYMG3HTTTXz22WdUqVKFX375hd9++42DDjqoxGOpKnfeeedu+xVnw4YN/PLLL5x33nmAJRjn4m3yZMjMhH33hf79bfrO/X2gnJgrKPm+6y4rVmrSxJJDLEvEK0+C2JNBg3atg4CYpeOuXbsybtw4fv31V7p168aoUaPIyclhxowZVKtWjfT09IjzQBRV3H7hcz0AO48VzV2Jc+W1fTu88opVQF97LZxyCgwfbkNse5uL+OrePb5VpF7EVCCOvXC6devG6NGjGTduHF27dmXdunUccMABVKtWjUmTJrEk0p1LBMXtd+CBB/L777+zatUqtm3bxrvvvgtA3bp1adSoEW+FZl/ftm3bznkfnCutoo38hg+3CXgOOcR6Po8ZY/0ZqlSxqjtPDsnP7yDCxSkdt2rVig0bNtCwYUMaNGhA9+7d+etf/7pzDobweRtKDi/yftWqVaN///506NCBZs2a7XK8l19+mauvvpr+/ftTrVo1xo4dyyGHHBLz39GltkiN/Hr1sueZmZYszjzTWySlGkmVYoiMjAzNysraZd3cuXM5omB4RFcu/l5WbsVV0R10UOQB9FzyEJEZqpoR6TUvYnLO7VFxbetLaCPhUoAXMSWg77//nssuu2yXdTVq1GDatGkBReQqu733hvXrd1+fKkNtu8hSPkGoKpJkBaOtW7dmZkmd+ipYqhRDurL5738tOVStaiOJFkjVobZdoZQuYqpZsyarVq3yL7hyUFVWrVrlfSgqqffeg9694S9/sekxK8NQ265QSt9BNGrUiOzsbHJycoIOJanVrFmTRo0aBR2GC8BBB8Gf/2wD6u21FxQp+XQpLqUTRLVq1WjWrFnQYTiXdDZtsgHhjj4a3nkn6GhcUFK6iMk5V3obNtjw2//8Z9CRuKB5gnDO7ZSXZ0Nk/PADnHRS0NG4oKV0EZNzLnqq0KePVUw/+6z1jHaVm99BOOcAm8jnmWfgttsK58pylZsnCOccAI0bWyulBx4IOhKXKLyIyblKbssWqFULLrzQHs4ViOsdhIh0FpGfRGS+iPQrZpsLRWSOiMwWkVfC1l8uIvNCj8vjGadzldWiRdC8OYwdG3QkLhHF7Q5CRNKAp4DTgWxguoi8rapzwrZpDtwBnKiqa0TkgND6fYEBQAagwIzQvmviFa9zlc2aNdYJbvNmaN066GhcIornHUR7YL6qLlTV7cBo4Jwi2/QCnir44lfV30PrzwQ+UtXVodc+AjrHMVbnKpXt2+H882HBAnjzTYhyShJXycQzQTQEloUtZ4fWhTsMOExEvhSRqSLSuRT7IiJXiUiWiGT5cBrORUfVWilNmmTjK516atARuUQVzwQRaQjVoqPmVQWaAx2Bi4HhIlIvyn1R1aGqmqGqGfv7rOjORa1xYxg4EC69NOhIXCKLZyumbKBx2HIjYHmEbaaqai6wSER+whJGNpY0wvedHLdInasktm2DGjXgvvuCjsQlg3jeQUwHmotIMxGpDnQD3i6yzVtAJoCI7IcVOS0EPgDOEJF9RGQf4IzQOudcGX36KRx6KMyaFXQkLlnE7Q5CVfNE5Hrsiz0NeF5VZ4vIvUCWqr5NYSKYA+wAblXVVQAich+WZADuVdXV8YrVuVT3009w3nlw4IE+C5yLnqTKZDoZGRmalZUVdBjOJZycHDjuOBuldepUOOSQoCNyiUREZqhqRqTXvCe1cylsyxY45xxYvtxaLXlycKXhYzE5l8Ly861Y6eWX7S7CudLwOwjnUtT27TYr3Btv2DzSzpWW30E4l4KGDYMOHWDlSk8Oruw8QTiXYj74AK69Fho0gHr1go7GJTNPEM6lkO++gwsugCOPhDFjoKoXIrty8AThXIpYvhz+8hfYe29491376Vx5eIJwLkXk5UGjRvC//9lP58rLb0CdS3I7dlhFdJMm8NVXXintYsfvIJxLcjfdBN26FSYK52LFE4RzSezxx+GJJ2z47rS0oKNxqcYThHNJ6q237O7hvPPg4YeDjsalIk8QziWh6dPhkkvg2GNh5Eio4v/JLg78Y+VcEtq2DVq2hLffhtq1g47GpSpvxeRcEsnLs85vJ51kdxFeKe3iye8gnEtwo0ZBeroVI+29N1x4oa335ODizROEc8UJ/2ZOT7flAEK46ipYsgRUYetWGD8+kFBcJeQJwrlIin4zL1liyxX8zXzXXbB5867rtm+39c7FmycI5yKJ9M28eXOFfzMvXVq69c7FkicI5yJZsiTy+gr6Zl650maDa9Ik8uvFrXculjxBOBcuLw/69Cn+9Th/M2/fDv/5D/zxjzZN6KBBuzdjrV3b1jsXb3FNECLSWUR+EpH5ItIvwus9RCRHRGaGHj3DXntIRGaLyFwRGSLibTZcnK1fD1262NgVnTtX+DfzhAnQujXccguceKLNId29OwwdCk2bWqulpk1tuXv3uIXhXCFVjcsDSAMWAIcA1YFZQMsi2/QAnoyw7wnAl6FjpAFTgI4lna9du3bqXJktWqTaqpVq1aqqzz5r60aOVG3aVBVUa9a05Ti57jo7zWGHqf7vf3E7jXO7AbK0mO/VeN5BtAfmq+pCVd0OjAbOiXJfBWpiiaUGUA34LS5ROjdlCrRvD7/8Au+/b62VwC7TFy+Gvn1tuWvXmJ523brCevCzz4bBg+H77+HPf47paZwrs3gmiIbAsrDl7NC6os4Xke9EZJyINAZQ1SnAJGBF6PGBqs4tuqOIXCUiWSKSlZOTE/vfwKW+V16BzEyoW9cSxWmn7b5Np07WAWHq1JiccscOGD4cmjeHf//b1p11lhUtVa8ek1M4FxPxTBCR6gy0yPI7QLqqtgEmAiMARORQ4AigEZZUOonIKbsdTHWoqmaoasb+++8f0+BdilOFgQPtLqFDB5g2DVq0iLztKadYZ7lJk8p92s8/twH2evWCww+Hc6K9p3YuAPFMENlA47DlRsDy8A1UdZWqbgstDgPahZ6fB0xV1Y2quhF4DzgujrG6ymTLFhsK9Z57oEcP+OgjqF+/+O3r1YNjjil3gnjgAcs1OTnw6qvw2Wd2WOcSVTwTxHSguYg0E5HqQDfg7fANRKRB2GIXoKAYaSlwqohUFZFqwKlhrzlXdr/9ZkVGo0fDgw/C889HV66TmWlFUEU7z+3B5s2werU9P+ss6N8ffvrJZoDzdnku0cUtQahqHnA98AH25f6aqs4WkXtFpEtosz6hpqyzgD5YqyaAcVgLqO+x1k+zVPWdeMXqKonvv7fK6Fmz4PXX4fbbo/+WzsyE3Fyb9DkKqvDaa3DEEYV13G3b2k2LD8/tkkVch/tW1QnAhCLr+oc9vwO4I8J+O4Cr4xmbq2QmTICLLrLK6M8/h3bt9rxPuJNOsnG2P/kE/vSnEjf99ltLCp9/bkmhV69yxO1cgLwntUttqjBkCPz1r9Zs6OuvS58cwMbZPvbYPdZDjBhhh5871zq0ZWVZvYNzycgThEtdubnQu7ddznfpYpf0DSO1tI5Sp042S8+GDbus3r4dVqyw52ecAf/4B8ybZ3cOaWnliN+5gHmCcKlp7Vr4y1/gmWfgttuszqFOnfIdMzPTOjF8/vnOVe+/D23a2CQ+qtCgATz0kDV8ci7ZeYJwqWfhQjjhBJg82Vop/fvf1o+hvE44wVo8TZrEzz9b7+ezzrJRV/vtNtKYc8nP56R2qeWLL+Dcc+1y/qOP4NRTY3fsWrXg+ONZP34SRz5ui4MHww03eA9ol5r8DsKljpdesqEy6te3YTHKmRzCZxxt2hQefxzIzGTv+d/Q7+o1/PyzD4/hUpsnCJf88vPh7rvh8sutOerUqdZiqRyKzji6dCncdBO8tzUTUeXeP33GgQfGKH7nEpQnCJfcNm+2/g2DBlmzofffh332KfdhI804qgp9RnWwsqVPPin3OZxLdF4H4ZLXihXWfHXGDHjkEbvEj9H4FcXNLLoguwacdmJMBu5zLtH5HYRLTjNn2rAZc+fC+PFw880xHdyocePI65s0wfpDfP+9jbrnXArzBOF2Cq+UTU+35YT09ttW1wDw5ZfWSzpGXnwRli+Hf/2rhBlHMzNtxeTJMTuvc4nIE4QDdq+UXbLElhMqSahau9Jzz4WWLW3YjKOOismht2+Ha6+FK66w1kolzgXdrh3stZcXM7nUV9xcpMn28Dmpy6dJE5sTueijadOgIwvZtk21Z08LqmtX1U2bYnbo335TPflkO/Ttt6vm5UWx05//rHr44TGLwbkyKZg3XcR+lmHedEqYk9orqSu57dth5MjiK2WLW1+hVq+2+aAnTbLmrPfcE5ue0VgVxplnwsqVNvvoxRdHuWNmpo0Qu3w5HHxwTGJxrlQKbvsLmtsV3PZD6Fa3/LyIqZLr2hWuvBKqVYv8uio8+qh1Nagw4ZUhDRtCq1ZW1/DSS3DffTFLDmBjJx1+uHXAjjo5gNdDuOBFaou9ebOtjxFPEJXMmjVW0bpqlS3ffLN1HXjhhd0rZWvVsvkMbr4ZOnaEBQsqIMCilSHLl8Ovv9rkPpddFpNT7NgBTzwBW7faoHoffVSGqT/btrWdvT+EC0oF3PZHlSBE5DgR2TtseW8R6RCzKFzcFXzHNmlipTTvv2/rO3a0IpZIlbLDhsE331jLnu++s1FLZ8yIc6CRrorA7h5iYO1aG2SvTx8YO7YcB0pLs6E8vKLaBaVRo8jrmzSJ3TmKq5wIfwDfAhK2XAX4Jpp9K+rhldSR5eWpXnedao0aqlWqqF58seqsWaU/zrJlqrfcopqba8vbt8c2zp1EIteWi5T70HPmqDZvrlqtmuqzz8Yg1scft9gWL47BwZwrpc6dd/8/qV271BXVlFBJHW0Rk4QOVJBU8vFe2Alt+XL7mZYGv/0Gf/87/PSTVcS2aVP64zVqZC1Mq1a1/mFHHAHPPmufypgq7uqnnFdFEydChw6wbp2VChXU5ZVLQT2E30W4ivbll/DBBzb9bcS22LERbYJYKCJ9RKRa6NEXWBizKFzMTJtm3QSaNrVpEcCKUoYOhUMPjc05cnOhWTO45hornlq2LDbHBSLXM+zsoVZ2jRvbjKFZWYV97MqtVSvYbz9PEK5ibdkC//d/dtH05puweLG1Ilm8OKbJAaJPENcAJwC/ANlAByAW12AuBlTtCvm00+C44+Czz6wov2DMuhiOQAFYq84PP7TJ2r76Co480ublKffdRG4uvPGGfek2blzuq6JNmyxGVWup9PHHxQ+hUSZVqthdxCefxOFWyrliDBgAP/8Mw4dbh814Kq7sKRYPoDPwEzAf6Bfh9R5ADjAz9OgZ9loT4ENgLjAHSC/pXJW5DuLXX1WrV1dt0EB18GDV9esr7twLF6p27Kh67rmq+fnlPNhDD1k56jvvxCSuNm2s3mX69HIfrnhPP20xz5sXx5M4FzJ1qn2oe4siDvkAAB88SURBVPWK2SEpb0c5EXkB2O0SSVX/r4R90oCngNOxu47pIvK2qs4psukYVb0+wiFeAgap6kcishdQkS3xE1puLowebW33n30WDjzQmmp26AA1alRsLM2a2ZX5pk12wT9/PkyZApdeWso7l6VLYeBAOOcca2ZUDpMmwQUXWHPWCRMgI6NchytZp06FJ41VGZ5zkWzbZkVLBx8MDz9cIaeMtojpXeB/ocfHQF1g4x72aQ/MV9WFqrodGA2cE83JRKQlUFVVPwJQ1Y2qGqHtY+WyZQs8/bTNhfP3v9sX8bp19topp1R8cihQpQrsHWoE/eSTFtu551rT2qjdeKP9fPzxcsUybBicfjoccIAN1XTmmeU63J4ddpj1tvP+EC7e7r0X5syxD/kf/lAhp4wqQajq62GPUcCFwJF72K0hEF59mR1aV9T5IvKdiIwTkYIS4sOAtSLyhoh8KyIPh+5IdiEiV4lIlohk5aT40Mtff21X67172wXE22/biNcV9DmJ2iOP2OPDD60Od/ToKIrn//c/q2zr39/qHMqhcWMb3DUGk8pFR8TqISZN8noIFz8zZsC//w09ekDnzhV33uLKnkp6AIdjdwclbXMBMDxs+TLgiSLb1AdqhJ5fA3wSet4VWAccgjWnfR24sqTzpWIdxG+/qc6YYc/Xr1f9299UJ0+OQVl/BZg7V7VDByueHzashA03bVJt1ky1ZUsbkK8Mli9XfeWVssUZE8OH2y86e3aAQbiUtW2bauvWqgcfrLp6dcwPT3n7QYjIBhFZH3qsA94BbtvDbtlAeJuRRsDyIslplapuCy0OA9qF7futWvFUHvAWUNrBEJJG0XkYhgyxnr7p6VZco2pFOK+/bp13Y90qKR5atLA6kiefLBzjaM2aCBs+8AAsWmRlZ9Wrl/o8X39tdQzXXFM4fEiF8/4QLp4GDbIJqp59NibT6ZZKcZmj6APYF2veemroccoetq+K9ZVoBlQHZgGtimzTIOz5ecDU0PO00Pb7h5ZfAHqXdL5kvYMYOdI6P0bqONyjh12Jp4LNm2107IsuUl25MrTyxx+tW/Nll5XpmCNGWA/x9PSy9Q6Pmfx8Gy/9/PMDDMKlpG+/Va1aVfXSS+N2Ckq4g4g2OfQEvgfWAJOALYSKg/aw35+Bn4EFwF2hdfcCXULPHwBmh5LBJKBF2L6nA9+FzvsiUL2kcyVrgmjadPfkAKoNGwYdWWzl5qref7/lgwMPVH3rzXzV005TrVfP2umW0j/+Ye9Tp06qOTlxCLi0evRQ3Xdf1R07go7EpYrt21XbtrV/mFWr4naaWCSI74GawMzQcguseWqZ6jDi8UjGBLFjR+TkEKOhhxLSrFn2mb+IV1VBtz76dJmOM3iwat++hWNDBW7ECPvDzZwZdCQuVdx3n32m3ngjrqcpKUFEO57SVlXdKiKISA1V/VFEDo9yXxfBrFlw3XXFvx7LARkTSZs2MO3DdWw75CbmkkHza6PvkP/DDzau1Gmn2RDkCVUXU1AP8cknMZsG1VViP/xgzVovugjOOy+wMKLtB5EtIvWwyuKPRGQ8RSqcXXTWrbMm/8ccY73le/XafR6GGAw9lNCq39+fvTf9xmETn6FqjTRycqBv38I+HZG88YYNI3LDDdYBLqGSA1j72kMP9YpqV355eTY5er16NnFJkIq7tSjugVVQd2EPdQIV/UiWIqbPPrOe8tdeW1isGINpZZPHjBn2BvTuvXPVmDGqaWmqjRqpfvDBru9HkybWvBdU27dX/eWX4ELfo169VOvWTaByL5eUHnzQPvBjxlTI6SihiEns9eSXkZGhWVlZQYcR0Zw5NoDeNdfY8uLF1oS10snPh+OPt9nifvzRrpBCpk+Hyy+3OaKrVrWLqHAnn2yd72rWrOCYS2P0aGvT+/XXNnSsc6X14482W+Ff/gLjxlXIrbKIzFDViAPS+JSjcbRxI9x2mxVJ9+8P69fb+kqZHMBGn/z6a+tqHZYcwL5Pv/kG6tbdPTmA5ZSETg5g0/OBFzO5stmxw4qW6tSxfkEJUI7qCSIOVG0OhhYtbEytyy+3u4i6dYOOLEC//w79+tmX6CWXRNykZk3YsCHy7jGdcyJeDjrIZlLyBOHK4vHHbYyYJ56wETgTgCeIOMjOttFM99/f5ksYPtymOKjUbr/dbqn2cGUUpwnlKk6nTvD55zbkrnPR+vlnm8SlS5fCoQcSgCeIGNm8GV5+2Z43bmx1DllZVuRe6X3+Obz4IvzjH3aFXYJBg5K8VVdmpo19Pn160JG4ZJGfD1deabfQ//1vQhQtFfAEUU6qMH48tGxp4yZ9+62t79DB5oOu9HJz4dprbZTWu+/e4+bdu9sEcnGcZje+CuohfPhvF60nn7SByx57zIaOTyCeIMph4UIbWvrcc20wvU8/haOPDjqqBPP44zB7to1AWPTWoBjdu8d1mt34ql/fWiV4PYSLxoIFcMcdcNZZdoWZYDxBlFFenpUmfPqpNcr55hubtMeFWbbMZonr0sUelUVmplU+bd0adCQukeXnQ8+e1q576NCEKloq4AmilCZPtuRQtSqMGGHNlm++GapVCzqyBHTjjfZPUM5Z4pJOp06WHKZODToSl8iefda+UB55BBo1CjqaiDxBRGnpUvjb3+zi8KWXbF3HjtAw0hx5ziaDfuMN+Oc/K1/Hj1NOsck9vJjJFWfJEuskdfrpVkGdoDxB7MH27TanTYsW8MEH9vzSS4OOKsFt2WKDJrVoAbfcEnQ0Fe8Pf4B27byi2kWmaoOwgc0vnYBFSwWiHc210rroInjrLRtQ8bHHkqg9fpAeeMBq8D/5pEyzxKWEzEx49FFr/xxl5byrJJ57Dj76CJ55ptxzsMeb30FE8MsvhcNi3HprYWmJJ4co/PyzTa5+6aWFQ2BXRpmZ1sT3yy+DjsQlkmXL7K46MxOuin6o+6BU+gQRPh9006bWpLJFCxuKHeCEE6wFmouCKvTuDbVq2RgjldlJJ1lLBi9mcgVULSnk5dnwClUS/+u3UhcxjRplf6/Nm2156VJ45RUbTLGkyXxcMV57DSZOtI4/Bx0UdDTB2msvaN/eK6pdoREj4P33rU/QIYcEHU1UEj+FxdFddxUmh3CrVyfN3y9xrF8PN91klbMF45pXdpmZNt5KQXmlq7yWL7f/j5NPtrvsJFGpE8TSpZHXJ8XIoYmmf3/49VerePMxRkynTjaE8+efBx2JC5KqXTRt3WoV1ElQtFQgeSKNg6QfOTRRfPutDVF8zTU+UU6444+3VlxezFS5vfIKvPOOjTjZvHnQ0ZRKXBOEiHQWkZ9EZL6I9Ivweg8RyRGRmaFHzyKv1xWRX0TkyXjEl/QjhyaC/HwbjG+//fyNK6pWLWvl4BXVldevv0KfPnax0Ldv0NGUWtwShIikAU8BZwEtgYtFpGWETceoatvQY3iR1+4DPo1XjEk/cmgieO45mDYNBg+GffYJOprEk5kJM2daxZarXApa9W3aBM8/n5RFr/G8g2gPzFfVhaq6HRgNnBPtziLSDjgQ+DBO8QFJPnJo0HJybCKgU0/17uXFycy0L4rPPgs6ElfRxo61DlT33GNt55NQPBNEQyC8ujc7tK6o80XkOxEZJyKNAUSkCvAIcGtJJxCRq0QkS0SycnJyYhW3i9btt9scoQkyf25C6tDBipq8mKlyycmxu4djj03q4WbimSAifWNokeV3gHRVbQNMBEaE1l8HTFDVEtsTqepQVc1Q1Yz999+/3AG7UvjiC3jhBfvwt4xUcugAq6Q+6SSvqK5sbrjBmje/8IJ1mExS8UwQ2UDjsOVGwPLwDVR1lapuCy0OA9qFnh8PXC8ii4HBwN9F5ME4xupKo2CWuCZNbLRWV7LMTPjhB/j996AjcRXhjTdgzBhr+t2qVdDRlEs8E8R0oLmINBOR6kA34O3wDUQkfH69LsBcAFXtrqpNVDUd+Afwkqru1grKBWTIEPvCGzIE6tQJOprE16mT/Zw8OdAwXAVYtcouno4+2obzTnJxSxCqmgdcD3yAffG/pqqzReReESmYXqyPiMwWkVlAH6BHvOJxMbJsGQwYAGefXblmiSuPdu1sTlovZkp9fftai7UXXkiJWcREtWi1QHLKyMjQrKysoMNIfV272vC2s2dDs2ZBR5M8zj4b5s2Dn34KOhIXL++8YxdNAwbYVLtJQkRmqGpGpNcqdU9qV0rvvQevvw533+3JobQyM20o9F9+CToSFw9r1sDVV0ObNnDnnUFHEzOeIFx0tmyB66+Hww9P6mZ7gSmYG8OLmVLTzTdbI4QXXkipSbI8QbjoPPigzRL39NNQo0bQ0SSfo46ynuaeIFLPe+/Biy9av6Bjjgk6mpjyBOH27OefLUFccklhixxXOmlp1uPcE0RqCJ9p7K9/hYMPtmatKcYThCtZwXgyNWvCI48EHU1y69QJFi2yMV1c8iqYaWzJEvv/2LHDWi6NGxd0ZDHnCcKVbOxYmyVu0CCfJa68vB4iNUSaaWzrVlufYjxBuOKtXw833mjlqtdeG3Q0ya9VK9h/f08Qya64mcaKW5/EkneQEBd/AwbYePZvvZWUQxUnHBG7i/jkEyua8AEOk4uqzSstYs+LSsGZxvwOwkU2c6YNpXH11dC+fdDRpI7MTOsLMX9+0JG40liwAE4/Ha64Ag491OrkwqXoTGOeIFyh8JYZHTrYOEv/+lfQUaWWgnoIH/47OeTl2WRYrVvD9Onw3//C3LkwfHilmGnME4QzRVtmbN8O27bZsBoudg47zJpEej1E4vvmG7tQuvVWOOMMmDPH7qirVKk0M415gkgE4Vfu6em2HG+q1vJi5Uprennrrbu3zNi+PSVbZgSqoB5i0qTI5dgueJs320is7dvD8uXWfPXNN6FhpPnOUptXUget4Mq94Mt5yRJbhl2vSnbsgI0b7bFhQ8k/o90mL2/P8aVgy4zAdepkf/c5c5J+voCU8/HH9v+3cCH06gX//nelnmvdE0TQIrWp3rzZKsP69y/8Qt+yJfpj1q4Ne+1lQ0wX/Kxf3+5OwteHP7/1VrubKCoFW2YELrw/RDIliFGj7PO6dKl9LgYNSp2ildWr4R//sLGUmje3v03HjkFHFThPEEHJz7cJZJYsifx6bi4cf/yuX/Il/Sx4XqdO2ZqkVqu2650MpGzLjMA1a2YVm598YgMgJoNo73STjap1Br3hBksSd9xhsyTWqhV0ZIlBVVPi0a5dO00Kc+ao9uun2rixKqiK2M+ij6ZNKz62kSPtvCL2c+TIio+hsrjiCtV991XdsSPoSKLTtGnifE5jZelS1bPPtt8jI0N15sygIwoEkKXFfK96JXVFyMmBJ56AY4+Fli3h4Yet2dyrr8Jzz9mVerigrtwrScuMhJCZaVes330XdCTRKan3cH5+xcZSXvn58NRT9r/4ySfwn//A1Kk24q7bhRcxxcvWrfDuu/DSSzYccF4etG1rH8aLL951XKPq1VO3bNdFFt4fom3bYGOJRv36keuoVO0ze+GF0K2bXQQlcg/x2bOt8nnKFGu6+t//+uRXJfApR2NJ1T54L70EY8bA2rXQoAFceilcdpndNThX4LDDbAKmd94JOpKS/fyzzZSWm7vr3UKtWtaYIjvbLoJyc+3L9qKL7HHUUYmTLLZtgwcesI6fdevCY4/ZRViixBegkqYcDbzuIFaPQOsgFixQHThQ9Y9/tPLM2rVVu3dX/eAD1by84OJyie2qq1Tr1lXNzQ06kuJt3qzapo1q/fqqjz9efB3VmjWqL7ygeuaZqmlp9n9w+OGq/ftbvVuQvvxS9YgjLKbu3VV//z3YeBIMJdRBBP7FHqtHhSeINWtUhw5VPekk3VnZ3KmT6osvqq5fX7GxuOQ0erR9dqZNCzqS4vXsaTG+9170++TkqP73v6qZmYWNMFq3Vh00SHX+/PjFWtS6darXXVeY0CZMqLhzJ5HAEgTQGfgJmA/0i/B6DyAHmBl69AytbwtMAWYD3wEX7elcFZIgtm9Xfecd1QsuUK1Rw96+Fi1U//Uv1SVL4n9+l1p+/dU+Qw8+GHQkkY0YYfHdeWfZj7F8ueqQIaonnqg7Wz5lZKg+/HB8/2fGj1dt2NCSw403qm7YEL9zJblAEgSQBiwADgGqA7OAlkW26QE8GWHfw4DmoecHAyuAeiWdL24JIj9fdcYM1b59Vfff396y/fZTveEG1enT7XXnyqplS9Uzzgg6it3Nnm1FpaeeGrsisCVLVAcPtgRRkCxOOMESyIoVsTnHihV2AVdw1zJ1amyOm8KCShDHAx+ELd8B3FFkm4gJIsKxZhUkjOIeZU4QxbX9X7bMruxatrS3qXp11a5d7cpk27ayncu5oq6/3r6IE+kztXGjldkfcIDdAcTD/PlW5NSmje4sos3MtKKpnJzSHy8/X/W551Tr1bO7+0GD7I7f7VFQCaIrMDxs+bKiySCUIFaEipHGAY0jHKc9MBeoEuG1q4AsIKtJkyalf2dGjrR/zvCOP9Wrq7ZqVVh2esIJ9qFdvbr0x3duT15/3T5nX3wRdCQmP1/1ssvs8z9xYsWcc84c1QEDrFIbrJL7zDOt0nvNmj3vP2+e1f+B6imnqP74Y7wjTilBJYgLIiSIJ4psUx+oEXp+DfBJkdcbhOowjtvT+cp0B1Fc79C0NGt9MW9e6Y/pXGmsXGlfxvfeG3QkZtgw+x+4556KP3d+vvVm7tdPtVmzwgu2Ll1UR42yeoTwO/4mTVQvuki1Zk3VP/xB9dlnk6dnegIpKUHErR+EiBwPDFTVM0PLdwCo6gPFbJ8GrFbVP4SW6wKTgQdUdeyezlemfhBVqkQeclkk+XqHuuR19NE2YmjQkwjNmgXHHQcnn2z9GoKcZlbVJugZM8Yev/xi44Xl59vIxuEyMmD8eJtnw5VaSf0g4jnUxnSguYg0E5HqQDfg7SKBNQhb7IIVJRHa/k3gpWiSQ5kVN1Kpj2DqKlJmJnz1lfW+D8r69XDBBbDvvjByZPBzkIvYfAyPPGIjDHz+OdSosXtyABvKxpNDXMQtQahqHnA98AH2xf+aqs4WkXtFpEtosz4iMltEZgF9sDoJgAuBU4AeIjIz9Ij9eASDBiXOOEiu8urUyXr6TpkSzPlVbfiJhQttfLADDggmjuJUqQInnQSbNkV+3ecsiZu4jsWkqhOACUXW9Q97fgfWuqnofiOBkfGMDSgc78jHQXJBOvlk+xKcNKlwjKaK9Mwz8Npr8OCDcMopFX/+aDVpEnl4fL/jjxsfi8m5RNC+vQ3a+MUXFXverCw48UQ4/XR4+21LVImq6JwUYHf8Q4f6RV05BFUH4ZyLVqdOMG1a8cUo8bB2rY3CeuCBMGJEYicHsCQwdKhNtiRiPz05xFWCfyKcqyQyM21I+Iq6g1C1kViXLbPipfr1K+a85eVzllQoTxDOJYKTToKqVa0eoiI89hi89ZZNXnXccRVzTpd0PEE4lwjq1IEOHSomQUydCrfdBuedB337xv98Lml5gnAuUWRmWqXxunXxO8eqVVbv0LgxPP+8T5jjSuQJwrlE0amTla1//nl8jp+fD3//O/z2G4wdC/Xqxec8LmV4gnAuURx/vPUWjlcx00MPwYQJ8Oij0K5dfM7hUoonCOcSRc2aliTiMSbTZ5/B3XfbXNHXXhv747uU5AnCuUTSqZMNmrdqVeyO+fvv0K0b/PGPMGyY1zu4qHmCcC6RZGZaH4VPP43N8XbssL4Ca9ZYvcPee8fmuK5S8AThXCJp396Gj4hVPcSgQTBxIjz5JLRpE5tjukrDE4RziaR6des0F4sE8fHHMHCgtVz6v/8r//FcpeMJwrlEk5kJs2dbc9SyWrECLrkEjjgCnn7a6x1cmXiCcC7RdOpkPydPLtv+eXlw8cWwcaPVO9SpE7PQXOXiCcK5RHPMMVaZXNbmrgMGWCX3s89Cy5axjc1VKp4gnEs0VavCqaeWrR7ivffgX/+yGeIuvTT2sblKxROEc4koMxPmzYPs7Oj3WbYMLrsMjjoKHn88frG5SsMThHOJqGDq0WjvInJzrZf09u02v0OtWvGLzVUaniCcS0RHHQX77BN9grjjDpgyBYYPh8MOi29srtKIa4IQkc4i8pOIzBeRfhFe7yEiOSIyM/ToGfba5SIyL/S4PJ5xOpdwqlSBjh2jSxDjx8Mjj0Dv3jaUt3MxErcEISJpwFPAWUBL4GIRidSkYoyqtg09hof23RcYAHQA2gMDRGSfeMXqXELKzLRpNRctKn6bRYugRw8bnfWRRyoqMldJxPMOoj0wX1UXqup2YDRwTpT7ngl8pKqrVXUN8BHQOU5xOpeYCvpDFHcXsW2b3TGoWn+HGjUqLjZXKcQzQTQEloUtZ4fWFXW+iHwnIuNEpHEp93UudbVsCQccUHx/iFtvtRnoRoyAZs0qNjZXKcQzQUTq269Flt8B0lW1DTARGFGKfRGRq0QkS0SycnJyyhWscwlHxIqZJk2yu4RwY8fCE0/AzTfDOdHemDtXOvFMENlA47DlRsDy8A1UdZWqbgstDgPaRbtvaP+hqpqhqhn7779/zAJ3LmFkZsLy5dYnosC8eXDllTa50IMPBhebS3nxTBDTgeYi0kxEqgPdgLfDNxCRBmGLXYC5oecfAGeIyD6hyukzQuucq1wK+kMUFDNt2WL1DtWrw5gxUK1acLG5lBe3BKGqecD12Bf7XOA1VZ0tIveKSJfQZn1EZLaIzAL6AD1C+64G7sOSzHTg3tA65yqX5s2hYcPCiuobb4SZM+Hll6Fx45L3da6cRIuWbSapjIwMzcrKCjoM52LvpJOsE1x+vi136WJ9H5yLARGZoaoZkV7zntTOJbJRo2D69MLkADZD3KhRwcXkKg1PEM4lsrvusvGVwm3ebOudizNPEM4lsqVLS7feuRjyBOFcImvSpHTrnYshTxDOJbJBg6B27V3X1a5t652LM08QziWy7t1h6FBo2tR6VjdtasvduwcdmasEqgYdgHNuD7p394TgAuF3EM455yLyBOGccy4iTxDOOeci8gThnHMuIk8QzjnnIkqZwfpEJAdYEnQc5bQfsDLoIBKIvx+78vejkL8XuyrP+9FUVSNOqJMyCSIViEhWcaMqVkb+fuzK349C/l7sKl7vhxcxOeeci8gThHPOuYg8QSSWoUEHkGD8/diVvx+F/L3YVVzeD6+DcM45F5HfQTjnnIvIE4RzzrmIPEEkABFpLCKTRGSuiMwWkb5BxxQ0EUkTkW9F5N2gYwmaiNQTkXEi8mPoM3J80DEFSURuCv2f/CAir4pIzaBjqkgi8ryI/C4iP4St21dEPhKReaGf+8TiXJ4gEkMecIuqHgEcB/QWkZYBxxS0vsDcoINIEI8D76tqC+AoKvH7IiINgT5AhqoeCaQB3YKNqsK9CHQusq4f8LGqNgc+Di2XmyeIBKCqK1T1m9DzDdgXQMNgowqOiDQC/gIMDzqWoIlIXeAU4DkAVd2uqmuDjSpwVYFaIlIVqA0sDzieCqWqnwGri6w+BxgRej4CODcW5/IEkWBEJB04GpgWbCSBegy4DcgPOpAEcAiQA7wQKnIbLiJ1gg4qKKr6CzAYWAqsANap6ofBRpUQDlTVFWAXnMABsTioJ4gEIiJ7Aa8DN6rq+qDjCYKInA38rqozgo4lQVQFjgGeUdWjgU3EqPggGYXK1s8BmgEHA3VE5NJgo0pdniAShIhUw5LDKFV9I+h4AnQi0EVEFgOjgU4iMjLYkAKVDWSrasEd5TgsYVRWfwIWqWqOquYCbwAnBBxTIvhNRBoAhH7+HouDeoJIACIiWBnzXFX9T9DxBElV71DVRqqajlU+fqKqlfYKUVV/BZaJyOGhVacBcwIMKWhLgeNEpHbo/+Y0KnGlfZi3gctDzy8HxsfioFVjcRBXbicClwHfi8jM0Lo7VXVCgDG5xHEDMEpEqgMLgSsCjicwqjpNRMYB32Ct/76lkg27ISKvAh2B/UQkGxgAPAi8JiJXYkn0gpicy4facM45F4kXMTnnnIvIE4RzzrmIPEE455yLyBOEc865iDxBOOeci8gThHN7ICI7RGRm2CNmPZlFJD18VE7nEon3g3Buz7aoatugg3CuovkdhHNlJCKLReTfIvJ16HFoaH1TEflYRL4L/WwSWn+giLwpIrNCj4IhItJEZFhojoMPRaRWaPs+IjIndJzRAf2arhLzBOHcntUqUsR0Udhr61W1PfAkNgotoecvqWobYBQwJLR+CPCpqh6Fjac0O7S+OfCUqrYC1gLnh9b3A44OHeeaeP1yzhXHe1I7twcislFV94qwfjHQSVUXhgZb/FVV64vISqCBquaG1q9Q1f1EJAdopKrbwo6RDnwUmugFEbkdqKaq94vI+8BG4C3gLVXdGOdf1bld+B2Ec+WjxTwvbptItoU930Fh3eBfgKeAdsCM0AQ5zlUYTxDOlc9FYT+nhJ5/ReE0mN2BL0LPPwauhZ1zbtct7qAiUgVorKqTsMmT6gG73cU4F09+ReLcntUKG2UXbH7ogqauNURkGnaxdXFoXR/geRG5FZsNrmD01b7A0NCImzuwZLGimHOmASNF5A+AAI/6VKOuonkdhHNlFKqDyFDVlUHH4lw8eBGTc865iPwOwjnnXER+B+Gccy4iTxDOOeci8gThnHMuIk8QzjnnIvIE4ZxzLqL/B0z/6jSXcEycAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def plot_metric(dfhistory, metric):\n",
    "    train_metrics = dfhistory[metric]\n",
    "    val_metrics = dfhistory['val_'+metric]\n",
    "    epochs = range(1, len(train_metrics) + 1)\n",
    "    plt.plot(epochs, train_metrics, 'bo--')\n",
    "    plt.plot(epochs, val_metrics, 'ro-')\n",
    "    plt.title('Training and validation '+ metric)\n",
    "    plt.xlabel(\"Epochs\")\n",
    "    plt.ylabel(metric)\n",
    "    plt.legend([\"train_\"+metric, 'val_'+metric])\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "# 观察损失和准确率的变化\n",
    "plot_metric(dfhistory,\"loss\")\n",
    "plot_metric(dfhistory,\"auc\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T12:00:54.385245Z",
     "start_time": "2020-11-30T12:00:54.327398Z"
    }
   },
   "outputs": [],
   "source": [
    "# 预测\n",
    "y_pred_probs = model(torch.tensor(test_x).float())\n",
    "y_pred = torch.where(y_pred_probs>0.5, torch.ones_like(y_pred_probs), torch.zeros_like(y_pred_probs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T12:00:56.737988Z",
     "start_time": "2020-11-30T12:00:56.718007Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.]])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred.data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T12:05:44.173847Z",
     "start_time": "2020-11-30T12:05:44.138942Z"
    }
   },
   "outputs": [],
   "source": [
    "# 模型的保存与使用\n",
    "torch.save(model, './model/DCN.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T12:06:00.677755Z",
     "start_time": "2020-11-30T12:06:00.653782Z"
    }
   },
   "outputs": [],
   "source": [
    "net_clone = torch.load('./model/DCN.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T12:06:29.494665Z",
     "start_time": "2020-11-30T12:06:29.436819Z"
    }
   },
   "outputs": [],
   "source": [
    "y_pred_probs = net_clone(torch.tensor(test_x).float())\n",
    "y_pred = torch.where(y_pred_probs>0.5, torch.ones_like(y_pred_probs), torch.zeros_like(y_pred_probs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T12:06:37.649858Z",
     "start_time": "2020-11-30T12:06:37.629911Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.]])"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
